src/plugins/nat/nat44-ei/nat44_ei.c

   1 /*
   2  * nat44_ei.c - nat44 endpoint dependent plugin
   3  *
   4  * Copyright (c) 2020 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  * License for the specific language governing permissions and limitations
  15  * under the License.
  16  */
  17
  18 #include <vnet/plugin/plugin.h>
  19 #include <vpp/app/version.h>
  20
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ip/ip4.h>
  24 #include <vnet/ip/ip_table.h>
  25 #include <vnet/ip/reass/ip4_sv_reass.h>
  26 #include <vnet/fib/fib_table.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <vnet/plugin/plugin.h>
  29
  30 // nat lib
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/nat_inlines.h>
  34 #include <nat/lib/ipfix_logging.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_dpo.h>
  37 #include <nat/nat44-ei/nat44_ei_inlines.h>
  38 #include <nat/nat44-ei/nat44_ei.h>
  39
  40 nat44_ei_main_t nat44_ei_main;
  41
  42 extern vlib_node_registration_t nat44_ei_hairpinning_node;
  43 extern vlib_node_registration_t nat44_ei_hairpin_dst_node;
  44 extern vlib_node_registration_t
  45   nat44_ei_in2out_hairpinning_finish_ip4_lookup_node;
  46 extern vlib_node_registration_t
  47   nat44_ei_in2out_hairpinning_finish_interface_output_node;
  48
  49 #define skip_if_disabled()                                                    \
  50   do                                                                          \
  51     {                                                                         \
  52       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  53       if (PREDICT_FALSE (!nm->enabled))                                       \
  54         return;                                                               \
  55     }                                                                         \
  56   while (0)
  57
  58 #define fail_if_enabled()                                                     \
  59   do                                                                          \
  60     {                                                                         \
  61       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  62       if (PREDICT_FALSE (nm->enabled))                                        \
  63         {                                                                     \
  64           nat44_ei_log_err ("plugin enabled");                                \
  65           return 1;                                                           \
  66         }                                                                     \
  67     }                                                                         \
  68   while (0)
  69
  70 #define fail_if_disabled()                                                    \
  71   do                                                                          \
  72     {                                                                         \
  73       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  74       if (PREDICT_FALSE (!nm->enabled))                                       \
  75         {                                                                     \
  76           nat44_ei_log_err ("plugin disabled");                               \
  77           return 1;                                                           \
  78         }                                                                     \
  79     }                                                                         \
  80   while (0)
  81
  82 /* Hook up input features */
  83 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-ei-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  87                                "ip4-sv-reassembly-feature"),
  88 };
  89 VNET_FEATURE_INIT (ip4_nat44_ei_in2out, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-ei-in2out",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  93                                "ip4-sv-reassembly-feature"),
  94 };
  95 VNET_FEATURE_INIT (ip4_nat44_ei_out2in, static) = {
  96   .arc_name = "ip4-unicast",
  97   .node_name = "nat44-ei-out2in",
  98   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  99                                "ip4-sv-reassembly-feature",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output, static) = {
 103   .arc_name = "ip4-output",
 104   .node_name = "nat44-ei-in2out-output",
 105   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 106                                "ip4-sv-reassembly-output-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_fast, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-ei-in2out-fast",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 112                                "ip4-sv-reassembly-feature"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_fast, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-ei-out2in-fast",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 118                                "ip4-sv-reassembly-feature",
 119                                "ip4-dhcp-client-detect"),
 120 };
 121 VNET_FEATURE_INIT (ip4_nat44_ei_hairpin_dst, static) = {
 122   .arc_name = "ip4-unicast",
 123   .node_name = "nat44-ei-hairpin-dst",
 124   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 125                                "ip4-sv-reassembly-feature"),
 126 };
 127 VNET_FEATURE_INIT (ip4_nat44_ei_hairpin_src, static) = {
 128   .arc_name = "ip4-output",
 129   .node_name = "nat44-ei-hairpin-src",
 130   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 131                                "ip4-sv-reassembly-output-feature"),
 132 };
 133 VNET_FEATURE_INIT (ip4_nat44_ei_hairpinning, static) = {
 134   .arc_name = "ip4-local",
 135   .node_name = "nat44-ei-hairpinning",
 136   .runs_before = VNET_FEATURES ("ip4-local-end-of-arc"),
 137 };
 138 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_worker_handoff, static) = {
 139   .arc_name = "ip4-unicast",
 140   .node_name = "nat44-ei-in2out-worker-handoff",
 141   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 142 };
 143 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_worker_handoff, static) = {
 144   .arc_name = "ip4-unicast",
 145   .node_name = "nat44-ei-out2in-worker-handoff",
 146   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 147                                "ip4-dhcp-client-detect"),
 148 };
 149 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output_worker_handoff, static) = {
 150   .arc_name = "ip4-output",
 151   .node_name = "nat44-ei-in2out-output-worker-handoff",
 152   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 153                                "ip4-sv-reassembly-output-feature"),
 154 };
 155
 156 VLIB_PLUGIN_REGISTER () = {
 157   .version = VPP_BUILD_VER,
 158   .description = "IPv4 Endpoint-Independent NAT (NAT44 EI)",
 159 };
 160
 161 #define foreach_nat44_ei_classify_error                                       \
 162   _ (NEXT_IN2OUT, "next in2out")                                              \
 163   _ (NEXT_OUT2IN, "next out2in")                                              \
 164   _ (FRAG_CACHED, "fragment cached")
 165
 166 typedef enum
 167 {
 168 #define _(sym, str) NAT44_EI_CLASSIFY_ERROR_##sym,
 169   foreach_nat44_ei_classify_error
 170 #undef _
 171     NAT44_EI_CLASSIFY_N_ERROR,
 172 } nat44_ei_classify_error_t;
 173
 174 static char *nat44_ei_classify_error_strings[] = {
 175 #define _(sym, string) string,
 176   foreach_nat44_ei_classify_error
 177 #undef _
 178 };
 179
 180 typedef enum
 181 {
 182   NAT44_EI_CLASSIFY_NEXT_IN2OUT,
 183   NAT44_EI_CLASSIFY_NEXT_OUT2IN,
 184   NAT44_EI_CLASSIFY_NEXT_DROP,
 185   NAT44_EI_CLASSIFY_N_NEXT,
 186 } nat44_ei_classify_next_t;
 187
 188 typedef struct
 189 {
 190   u8 next_in2out;
 191   u8 cached;
 192 } nat44_ei_classify_trace_t;
 193
 194 void nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len,
 195                                    u32 sw_if_index, int is_add);
 196
 197 static u8 *
 198 format_nat44_ei_classify_trace (u8 *s, va_list *args)
 199 {
 200   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 201   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 202   nat44_ei_classify_trace_t *t = va_arg (*args, nat44_ei_classify_trace_t *);
 203   char *next;
 204
 205   if (t->cached)
 206     s = format (s, "nat44-ei-classify: fragment cached");
 207   else
 208     {
 209       next = t->next_in2out ? "nat44-ei-in2out" : "nat44-ei-out2in";
 210       s = format (s, "nat44-ei-classify: next %s", next);
 211     }
 212
 213   return s;
 214 }
 215
 216 static void nat44_ei_db_free ();
 217
 218 static void nat44_ei_db_init (u32 translations, u32 translation_buckets,
 219                               u32 user_buckets);
 220
 221 static void nat44_ei_ip4_add_del_interface_address_cb (
 222   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 223   u32 address_length, u32 if_address_index, u32 is_delete);
 224
 225 static void nat44_ei_ip4_add_del_addr_only_sm_cb (
 226   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 227   u32 address_length, u32 if_address_index, u32 is_delete);
 228
 229 static void nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque,
 230                                          u32 sw_if_index, u32 new_fib_index,
 231                                          u32 old_fib_index);
 232
 233 void
 234 nat44_ei_set_node_indexes (nat44_ei_main_t *nm, vlib_main_t *vm)
 235 {
 236   vlib_node_t *node;
 237   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-out2in");
 238   nm->out2in_node_index = node->index;
 239   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out");
 240   nm->in2out_node_index = node->index;
 241   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out-output");
 242   nm->in2out_output_node_index = node->index;
 243 }
 244
 245 int
 246 nat44_ei_set_workers (uword *bitmap)
 247 {
 248   nat44_ei_main_t *nm = &nat44_ei_main;
 249   int i, j = 0;
 250
 251   if (nm->num_workers < 2)
 252     return VNET_API_ERROR_FEATURE_DISABLED;
 253
 254   if (clib_bitmap_last_set (bitmap) >= nm->num_workers)
 255     return VNET_API_ERROR_INVALID_WORKER;
 256
 257   vec_free (nm->workers);
 258   clib_bitmap_foreach (i, bitmap)
 259     {
 260       vec_add1 (nm->workers, i);
 261       nm->per_thread_data[nm->first_worker_index + i].snat_thread_index = j;
 262       nm->per_thread_data[nm->first_worker_index + i].thread_index = i;
 263       j++;
 264     }
 265
 266   nm->port_per_thread = (0xffff - 1024) / _vec_len (nm->workers);
 267
 268   return 0;
 269 }
 270
 271 #define nat_validate_simple_counter(c, i)                                     \
 272   do                                                                          \
 273     {                                                                         \
 274       vlib_validate_simple_counter (&c, i);                                   \
 275       vlib_zero_simple_counter (&c, i);                                       \
 276     }                                                                         \
 277   while (0);
 278
 279 #define nat_init_simple_counter(c, n, sn)                                     \
 280   do                                                                          \
 281     {                                                                         \
 282       c.name = n;                                                             \
 283       c.stat_segment_name = sn;                                               \
 284       nat_validate_simple_counter (c, 0);                                     \
 285     }                                                                         \
 286   while (0);
 287
 288 static_always_inline void
 289 nat_validate_interface_counters (nat44_ei_main_t *nm, u32 sw_if_index)
 290 {
 291 #define _(x)                                                                  \
 292   nat_validate_simple_counter (nm->counters.fastpath.in2out.x, sw_if_index);  \
 293   nat_validate_simple_counter (nm->counters.fastpath.out2in.x, sw_if_index);  \
 294   nat_validate_simple_counter (nm->counters.slowpath.in2out.x, sw_if_index);  \
 295   nat_validate_simple_counter (nm->counters.slowpath.out2in.x, sw_if_index);
 296   foreach_nat_counter;
 297 #undef _
 298   nat_validate_simple_counter (nm->counters.hairpinning, sw_if_index);
 299 }
 300
 301 clib_error_t *
 302 nat44_ei_init (vlib_main_t *vm)
 303 {
 304   nat44_ei_main_t *nm = &nat44_ei_main;
 305   vlib_thread_main_t *tm = vlib_get_thread_main ();
 306   vlib_thread_registration_t *tr;
 307   ip4_add_del_interface_address_callback_t cbi = { 0 };
 308   ip4_table_bind_callback_t cbt = { 0 };
 309   u32 i, num_threads = 0;
 310   uword *p, *bitmap = 0;
 311
 312   clib_memset (nm, 0, sizeof (*nm));
 313
 314   // required
 315   nm->vnet_main = vnet_get_main ();
 316   // convenience
 317   nm->ip4_main = &ip4_main;
 318   nm->api_main = vlibapi_get_main ();
 319   nm->ip4_lookup_main = &ip4_main.lookup_main;
 320
 321   // handoff stuff
 322   nm->fq_out2in_index = ~0;
 323   nm->fq_in2out_index = ~0;
 324   nm->fq_in2out_output_index = ~0;
 325   nm->worker_in2out_cb = nat44_ei_get_in2out_worker_index;
 326   nm->worker_out2in_cb = nat44_ei_get_out2in_worker_index;
 327
 328   nm->log_level = NAT_LOG_ERROR;
 329
 330   nat44_ei_set_node_indexes (nm, vm);
 331   nm->log_class = vlib_log_register_class ("nat44-ei", 0);
 332
 333   nat_init_simple_counter (nm->total_users, "total-users",
 334                            "/nat44-ei/total-users");
 335   nat_init_simple_counter (nm->total_sessions, "total-sessions",
 336                            "/nat44-ei/total-sessions");
 337   nat_init_simple_counter (nm->user_limit_reached, "user-limit-reached",
 338                            "/nat44-ei/user-limit-reached");
 339
 340 #define _(x)                                                                  \
 341   nat_init_simple_counter (nm->counters.fastpath.in2out.x, #x,                \
 342                            "/nat44-ei/in2out/fastpath/" #x);                  \
 343   nat_init_simple_counter (nm->counters.fastpath.out2in.x, #x,                \
 344                            "/nat44-ei/out2in/fastpath/" #x);                  \
 345   nat_init_simple_counter (nm->counters.slowpath.in2out.x, #x,                \
 346                            "/nat44-ei/in2out/slowpath/" #x);                  \
 347   nat_init_simple_counter (nm->counters.slowpath.out2in.x, #x,                \
 348                            "/nat44-ei/out2in/slowpath/" #x);
 349   foreach_nat_counter;
 350 #undef _
 351   nat_init_simple_counter (nm->counters.hairpinning, "hairpinning",
 352                            "/nat44-ei/hairpinning");
 353
 354   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 355   if (p)
 356     {
 357       tr = (vlib_thread_registration_t *) p[0];
 358       if (tr)
 359         {
 360           nm->num_workers = tr->count;
 361           nm->first_worker_index = tr->first_index;
 362         }
 363     }
 364   num_threads = tm->n_vlib_mains - 1;
 365   nm->port_per_thread = 0xffff - 1024;
 366   vec_validate (nm->per_thread_data, num_threads);
 367
 368   /* Use all available workers by default */
 369   if (nm->num_workers > 1)
 370     {
 371
 372       for (i = 0; i < nm->num_workers; i++)
 373         bitmap = clib_bitmap_set (bitmap, i, 1);
 374       nat44_ei_set_workers (bitmap);
 375       clib_bitmap_free (bitmap);
 376     }
 377   else
 378     nm->per_thread_data[0].snat_thread_index = 0;
 379
 380   /* callbacks to call when interface address changes. */
 381   cbi.function = nat44_ei_ip4_add_del_interface_address_cb;
 382   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 383   cbi.function = nat44_ei_ip4_add_del_addr_only_sm_cb;
 384   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 385
 386   /* callbacks to call when interface to table biding changes */
 387   cbt.function = nat44_ei_update_outside_fib;
 388   vec_add1 (nm->ip4_main->table_bind_callbacks, cbt);
 389
 390   nm->fib_src_low = fib_source_allocate (
 391     "nat44-ei-low", FIB_SOURCE_PRIORITY_LOW, FIB_SOURCE_BH_SIMPLE);
 392   nm->fib_src_hi = fib_source_allocate ("nat44-ei-hi", FIB_SOURCE_PRIORITY_HI,
 393                                         FIB_SOURCE_BH_SIMPLE);
 394
 395   // used only by out2in-dpo feature
 396   nat_dpo_module_init ();
 397   nat_ha_init (vm, nm->num_workers, num_threads);
 398
 399   nm->hairpinning_fq_index =
 400     vlib_frame_queue_main_init (nat44_ei_hairpinning_node.index, 0);
 401   nm->hairpin_dst_fq_index =
 402     vlib_frame_queue_main_init (nat44_ei_hairpin_dst_node.index, 0);
 403   nm->in2out_hairpinning_finish_ip4_lookup_node_fq_index =
 404     vlib_frame_queue_main_init (
 405       nat44_ei_in2out_hairpinning_finish_ip4_lookup_node.index, 0);
 406   nm->in2out_hairpinning_finish_interface_output_node_fq_index =
 407     vlib_frame_queue_main_init (
 408       nat44_ei_in2out_hairpinning_finish_interface_output_node.index, 0);
 409   return nat44_ei_api_hookup (vm);
 410 }
 411
 412 VLIB_INIT_FUNCTION (nat44_ei_init);
 413
 414 int
 415 nat44_ei_plugin_enable (nat44_ei_config_t c)
 416 {
 417   nat44_ei_main_t *nm = &nat44_ei_main;
 418
 419   fail_if_enabled ();
 420
 421   if (!c.users)
 422     c.users = 1024;
 423
 424   if (!c.sessions)
 425     c.sessions = 10 * 1024;
 426
 427   nm->rconfig = c;
 428
 429   if (!nm->frame_queue_nelts)
 430     nm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
 431
 432   nm->translations = c.sessions;
 433   nm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
 434   nm->user_buckets = nat_calc_bihash_buckets (c.users);
 435
 436   nm->pat = (!c.static_mapping_only ||
 437              (c.static_mapping_only && c.connection_tracking));
 438
 439   nm->static_mapping_only = c.static_mapping_only;
 440   nm->static_mapping_connection_tracking = c.connection_tracking;
 441   nm->out2in_dpo = c.out2in_dpo;
 442   nm->forwarding_enabled = 0;
 443   nm->mss_clamping = 0;
 444
 445   nm->max_users_per_thread = c.users;
 446   nm->max_translations_per_thread = c.sessions;
 447   nm->max_translations_per_user =
 448     c.user_sessions ? c.user_sessions : nm->max_translations_per_thread;
 449
 450   nm->inside_vrf_id = c.inside_vrf;
 451   nm->inside_fib_index = fib_table_find_or_create_and_lock (
 452     FIB_PROTOCOL_IP4, c.inside_vrf, nm->fib_src_hi);
 453
 454   nm->outside_vrf_id = c.outside_vrf;
 455   nm->outside_fib_index = fib_table_find_or_create_and_lock (
 456     FIB_PROTOCOL_IP4, c.outside_vrf, nm->fib_src_hi);
 457
 458   nat_reset_timeouts (&nm->timeouts);
 459   nat44_ei_db_init (nm->translations, nm->translation_buckets,
 460                     nm->user_buckets);
 461   nat44_ei_set_alloc_default ();
 462
 463   // TODO: zero simple counter for all counters missing
 464
 465   vlib_zero_simple_counter (&nm->total_users, 0);
 466   vlib_zero_simple_counter (&nm->total_sessions, 0);
 467   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
 468
 469   nat_ha_enable ();
 470   nm->enabled = 1;
 471
 472   return 0;
 473 }
 474
 475 void
 476 nat44_ei_addresses_free (nat44_ei_address_t **addresses)
 477 {
 478   nat44_ei_address_t *ap;
 479   vec_foreach (ap, *addresses)
 480     {
 481 #define _(N, i, n, s) vec_free (ap->busy_##n##_ports_per_thread);
 482       foreach_nat_protocol
 483 #undef _
 484     }
 485   vec_free (*addresses);
 486   *addresses = 0;
 487 }
 488
 489 int
 490 nat44_ei_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
 491 {
 492   const char *feature_name, *del_feature_name;
 493   nat44_ei_main_t *nm = &nat44_ei_main;
 494   nat44_ei_interface_t *i;
 495   nat44_ei_address_t *ap;
 496   nat44_ei_static_mapping_t *m;
 497   nat44_ei_outside_fib_t *outside_fib;
 498   u32 fib_index =
 499     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 500
 501   fail_if_disabled ();
 502
 503   if (nm->out2in_dpo && !is_inside)
 504     {
 505       nat44_ei_log_err ("error unsupported");
 506       return VNET_API_ERROR_UNSUPPORTED;
 507     }
 508
 509   pool_foreach (i, nm->output_feature_interfaces)
 510     {
 511       if (i->sw_if_index == sw_if_index)
 512         {
 513           nat44_ei_log_err ("error interface already configured");
 514           return VNET_API_ERROR_VALUE_EXIST;
 515         }
 516     }
 517
 518   if (nm->static_mapping_only && !(nm->static_mapping_connection_tracking))
 519     feature_name = is_inside ? "nat44-ei-in2out-fast" : "nat44-ei-out2in-fast";
 520   else
 521     {
 522       if (nm->num_workers > 1)
 523         feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 524                                    "nat44-ei-out2in-worker-handoff";
 525       else
 526         feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 527     }
 528
 529   if (nm->fq_in2out_index == ~0 && nm->num_workers > 1)
 530     nm->fq_in2out_index = vlib_frame_queue_main_init (nm->in2out_node_index,
 531                                                       nm->frame_queue_nelts);
 532
 533   if (nm->fq_out2in_index == ~0 && nm->num_workers > 1)
 534     nm->fq_out2in_index = vlib_frame_queue_main_init (nm->out2in_node_index,
 535                                                       nm->frame_queue_nelts);
 536
 537   if (!is_inside)
 538     {
 539       vec_foreach (outside_fib, nm->outside_fibs)
 540         {
 541           if (outside_fib->fib_index == fib_index)
 542             {
 543               if (is_del)
 544                 {
 545                   outside_fib->refcount--;
 546                   if (!outside_fib->refcount)
 547                     vec_del1 (nm->outside_fibs,
 548                               outside_fib - nm->outside_fibs);
 549                 }
 550               else
 551                 outside_fib->refcount++;
 552               goto feature_set;
 553             }
 554         }
 555       if (!is_del)
 556         {
 557           vec_add2 (nm->outside_fibs, outside_fib, 1);
 558           outside_fib->refcount = 1;
 559           outside_fib->fib_index = fib_index;
 560         }
 561     }
 562
 563 feature_set:
 564   pool_foreach (i, nm->interfaces)
 565     {
 566       if (i->sw_if_index == sw_if_index)
 567         {
 568           if (is_del)
 569             {
 570               if (nat44_ei_interface_is_inside (i) &&
 571                   nat44_ei_interface_is_outside (i))
 572                 {
 573                   if (is_inside)
 574                     i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 575                   else
 576                     i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 577
 578                   if (nm->num_workers > 1)
 579                     {
 580                       del_feature_name = "nat44-handoff-classify";
 581                       feature_name = !is_inside ?
 582                                        "nat44-ei-in2out-worker-handoff" :
 583                                        "nat44-ei-out2in-worker-handoff";
 584                     }
 585                   else
 586                     {
 587                       del_feature_name = "nat44-ei-classify";
 588                       feature_name =
 589                         !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 590                     }
 591
 592                   int rv =
 593                     ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 594                   if (rv)
 595                     return rv;
 596                   vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 597                                                sw_if_index, 0, 0, 0);
 598                   vnet_feature_enable_disable ("ip4-unicast", feature_name,
 599                                                sw_if_index, 1, 0, 0);
 600                   if (!is_inside)
 601                     {
 602                       vnet_feature_enable_disable ("ip4-local",
 603                                                    "nat44-ei-hairpinning",
 604                                                    sw_if_index, 1, 0, 0);
 605                     }
 606                 }
 607               else
 608                 {
 609                   int rv =
 610                     ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 611                   if (rv)
 612                     return rv;
 613                   vnet_feature_enable_disable ("ip4-unicast", feature_name,
 614                                                sw_if_index, 0, 0, 0);
 615                   pool_put (nm->interfaces, i);
 616                   if (is_inside)
 617                     {
 618                       vnet_feature_enable_disable ("ip4-local",
 619                                                    "nat44-ei-hairpinning",
 620                                                    sw_if_index, 0, 0, 0);
 621                     }
 622                 }
 623             }
 624           else
 625             {
 626               if ((nat44_ei_interface_is_inside (i) && is_inside) ||
 627                   (nat44_ei_interface_is_outside (i) && !is_inside))
 628                 return 0;
 629
 630               if (nm->num_workers > 1)
 631                 {
 632                   del_feature_name = !is_inside ?
 633                                        "nat44-ei-in2out-worker-handoff" :
 634                                        "nat44-ei-out2in-worker-handoff";
 635                   feature_name = "nat44-handoff-classify";
 636                 }
 637               else
 638                 {
 639                   del_feature_name =
 640                     !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 641                   feature_name = "nat44-ei-classify";
 642                 }
 643
 644               int rv =
 645                 ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 646               if (rv)
 647                 return rv;
 648               vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 649                                            sw_if_index, 0, 0, 0);
 650               vnet_feature_enable_disable ("ip4-unicast", feature_name,
 651                                            sw_if_index, 1, 0, 0);
 652               if (!is_inside)
 653                 {
 654                   vnet_feature_enable_disable (
 655                     "ip4-local", "nat44-ei-hairpinning", sw_if_index, 0, 0, 0);
 656                 }
 657               goto set_flags;
 658             }
 659
 660           goto fib;
 661         }
 662     }
 663
 664   if (is_del)
 665     {
 666       nat44_ei_log_err ("error interface couldn't be found");
 667       return VNET_API_ERROR_NO_SUCH_ENTRY;
 668     }
 669
 670   pool_get (nm->interfaces, i);
 671   i->sw_if_index = sw_if_index;
 672   i->flags = 0;
 673   nat_validate_interface_counters (nm, sw_if_index);
 674
 675   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
 676                                0);
 677
 678   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 679   if (rv)
 680     return rv;
 681
 682   if (is_inside && !nm->out2in_dpo)
 683     {
 684       vnet_feature_enable_disable ("ip4-local", "nat44-ei-hairpinning",
 685                                    sw_if_index, 1, 0, 0);
 686     }
 687
 688 set_flags:
 689   if (is_inside)
 690     {
 691       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 692       return 0;
 693     }
 694   else
 695     i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 696
 697   /* Add/delete external addresses to FIB */
 698 fib:
 699   vec_foreach (ap, nm->addresses)
 700     nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, !is_del);
 701
 702   pool_foreach (m, nm->static_mappings)
 703     {
 704       if (!(nat44_ei_is_addr_only_static_mapping (m)) ||
 705           (m->local_addr.as_u32 == m->external_addr.as_u32))
 706         continue;
 707
 708       nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 709                                     !is_del);
 710     }
 711
 712   return 0;
 713 }
 714
 715 int
 716 nat44_ei_interface_add_del_output_feature (u32 sw_if_index, u8 is_inside,
 717                                            int is_del)
 718 {
 719   nat44_ei_main_t *nm = &nat44_ei_main;
 720   nat44_ei_interface_t *i;
 721   nat44_ei_address_t *ap;
 722   nat44_ei_static_mapping_t *m;
 723   nat44_ei_outside_fib_t *outside_fib;
 724   u32 fib_index =
 725     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 726
 727   fail_if_disabled ();
 728
 729   if (nm->static_mapping_only && !(nm->static_mapping_connection_tracking))
 730     {
 731       nat44_ei_log_err ("error unsupported");
 732       return VNET_API_ERROR_UNSUPPORTED;
 733     }
 734
 735   pool_foreach (i, nm->interfaces)
 736     {
 737       if (i->sw_if_index == sw_if_index)
 738         {
 739           nat44_ei_log_err ("error interface already configured");
 740           return VNET_API_ERROR_VALUE_EXIST;
 741         }
 742     }
 743
 744   if (!is_inside)
 745     {
 746       vec_foreach (outside_fib, nm->outside_fibs)
 747         {
 748           if (outside_fib->fib_index == fib_index)
 749             {
 750               if (is_del)
 751                 {
 752                   outside_fib->refcount--;
 753                   if (!outside_fib->refcount)
 754                     vec_del1 (nm->outside_fibs,
 755                               outside_fib - nm->outside_fibs);
 756                 }
 757               else
 758                 outside_fib->refcount++;
 759               goto feature_set;
 760             }
 761         }
 762       if (!is_del)
 763         {
 764           vec_add2 (nm->outside_fibs, outside_fib, 1);
 765           outside_fib->refcount = 1;
 766           outside_fib->fib_index = fib_index;
 767         }
 768     }
 769
 770 feature_set:
 771   if (is_inside)
 772     {
 773       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
 774       if (rv)
 775         return rv;
 776       rv =
 777         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
 778       if (rv)
 779         return rv;
 780       vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-hairpin-dst",
 781                                    sw_if_index, !is_del, 0, 0);
 782       vnet_feature_enable_disable ("ip4-output", "nat44-ei-hairpin-src",
 783                                    sw_if_index, !is_del, 0, 0);
 784       goto fq;
 785     }
 786
 787   if (nm->num_workers > 1)
 788     {
 789       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
 790       if (rv)
 791         return rv;
 792       rv =
 793         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
 794       if (rv)
 795         return rv;
 796       vnet_feature_enable_disable ("ip4-unicast",
 797                                    "nat44-ei-out2in-worker-handoff",
 798                                    sw_if_index, !is_del, 0, 0);
 799       vnet_feature_enable_disable ("ip4-output",
 800                                    "nat44-ei-in2out-output-worker-handoff",
 801                                    sw_if_index, !is_del, 0, 0);
 802     }
 803   else
 804     {
 805       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
 806       if (rv)
 807         return rv;
 808       rv =
 809         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
 810       if (rv)
 811         return rv;
 812       vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
 813                                    sw_if_index, !is_del, 0, 0);
 814       vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
 815                                    sw_if_index, !is_del, 0, 0);
 816     }
 817
 818 fq:
 819   if (nm->fq_in2out_output_index == ~0 && nm->num_workers > 1)
 820     nm->fq_in2out_output_index =
 821       vlib_frame_queue_main_init (nm->in2out_output_node_index, 0);
 822
 823   if (nm->fq_out2in_index == ~0 && nm->num_workers > 1)
 824     nm->fq_out2in_index =
 825       vlib_frame_queue_main_init (nm->out2in_node_index, 0);
 826
 827   pool_foreach (i, nm->output_feature_interfaces)
 828     {
 829       if (i->sw_if_index == sw_if_index)
 830         {
 831           if (is_del)
 832             pool_put (nm->output_feature_interfaces, i);
 833           else
 834             return VNET_API_ERROR_VALUE_EXIST;
 835
 836           goto fib;
 837         }
 838     }
 839
 840   if (is_del)
 841     {
 842       nat44_ei_log_err ("error interface couldn't be found");
 843       return VNET_API_ERROR_NO_SUCH_ENTRY;
 844     }
 845
 846   pool_get (nm->output_feature_interfaces, i);
 847   i->sw_if_index = sw_if_index;
 848   i->flags = 0;
 849   nat_validate_interface_counters (nm, sw_if_index);
 850   if (is_inside)
 851     i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 852   else
 853     i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 854
 855   /* Add/delete external addresses to FIB */
 856 fib:
 857   if (is_inside)
 858     return 0;
 859
 860   vec_foreach (ap, nm->addresses)
 861     nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, !is_del);
 862
 863   pool_foreach (m, nm->static_mappings)
 864     {
 865       if (!((nat44_ei_is_addr_only_static_mapping (m))) ||
 866           (m->local_addr.as_u32 == m->external_addr.as_u32))
 867         continue;
 868
 869       nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 870                                     !is_del);
 871     }
 872
 873   return 0;
 874 }
 875
 876 int
 877 nat44_ei_plugin_disable ()
 878 {
 879   nat44_ei_main_t *nm = &nat44_ei_main;
 880   nat44_ei_interface_t *i, *vec;
 881   int error = 0;
 882
 883   // first unregister all nodes from interfaces
 884   vec = vec_dup (nm->interfaces);
 885   vec_foreach (i, vec)
 886     {
 887       if (nat44_ei_interface_is_inside (i))
 888         error = nat44_ei_interface_add_del (i->sw_if_index, 1, 1);
 889       if (nat44_ei_interface_is_outside (i))
 890         error = nat44_ei_interface_add_del (i->sw_if_index, 0, 1);
 891
 892       if (error)
 893         {
 894           nat44_ei_log_err ("error occurred while removing interface %u",
 895                             i->sw_if_index);
 896         }
 897     }
 898   vec_free (vec);
 899   nm->interfaces = 0;
 900
 901   vec = vec_dup (nm->output_feature_interfaces);
 902   vec_foreach (i, vec)
 903     {
 904       if (nat44_ei_interface_is_inside (i))
 905         error =
 906           nat44_ei_interface_add_del_output_feature (i->sw_if_index, 1, 1);
 907       if (nat44_ei_interface_is_outside (i))
 908         error =
 909           nat44_ei_interface_add_del_output_feature (i->sw_if_index, 0, 1);
 910
 911       if (error)
 912         {
 913           nat44_ei_log_err ("error occurred while removing interface %u",
 914                             i->sw_if_index);
 915         }
 916     }
 917   vec_free (vec);
 918   nm->output_feature_interfaces = 0;
 919
 920   nat_ha_disable ();
 921   nat44_ei_db_free ();
 922
 923   nat44_ei_addresses_free (&nm->addresses);
 924
 925   vec_free (nm->to_resolve);
 926   vec_free (nm->auto_add_sw_if_indices);
 927
 928   nm->to_resolve = 0;
 929   nm->auto_add_sw_if_indices = 0;
 930
 931   nm->forwarding_enabled = 0;
 932
 933   nm->enabled = 0;
 934   clib_memset (&nm->rconfig, 0, sizeof (nm->rconfig));
 935
 936   return error;
 937 }
 938
 939 int
 940 nat44_ei_set_outside_address_and_port (nat44_ei_address_t *addresses,
 941                                        u32 thread_index, ip4_address_t addr,
 942                                        u16 port, nat_protocol_t protocol)
 943 {
 944   nat44_ei_main_t *nm = &nat44_ei_main;
 945   nat44_ei_address_t *a = 0;
 946   u32 address_index;
 947   u16 port_host_byte_order = clib_net_to_host_u16 (port);
 948
 949   for (address_index = 0; address_index < vec_len (addresses); address_index++)
 950     {
 951       if (addresses[address_index].addr.as_u32 != addr.as_u32)
 952         continue;
 953
 954       a = addresses + address_index;
 955       switch (protocol)
 956         {
 957 #define _(N, j, n, s)                                                         \
 958   case NAT_PROTOCOL_##N:                                                      \
 959     if (a->busy_##n##_port_refcounts[port_host_byte_order])                   \
 960       return VNET_API_ERROR_INSTANCE_IN_USE;                                  \
 961     ++a->busy_##n##_port_refcounts[port_host_byte_order];                     \
 962     a->busy_##n##_ports_per_thread[thread_index]++;                           \
 963     a->busy_##n##_ports++;                                                    \
 964     return 0;
 965           foreach_nat_protocol
 966 #undef _
 967             default : nat_elog_info (nm, "unknown protocol");
 968           return 1;
 969         }
 970     }
 971
 972   return VNET_API_ERROR_NO_SUCH_ENTRY;
 973 }
 974
 975 void
 976 nat44_ei_add_del_address_dpo (ip4_address_t addr, u8 is_add)
 977 {
 978   nat44_ei_main_t *nm = &nat44_ei_main;
 979   dpo_id_t dpo_v4 = DPO_INVALID;
 980   fib_prefix_t pfx = {
 981     .fp_proto = FIB_PROTOCOL_IP4,
 982     .fp_len = 32,
 983     .fp_addr.ip4.as_u32 = addr.as_u32,
 984   };
 985
 986   if (is_add)
 987     {
 988       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
 989       fib_table_entry_special_dpo_add (0, &pfx, nm->fib_src_hi,
 990                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
 991       dpo_reset (&dpo_v4);
 992     }
 993   else
 994     {
 995       fib_table_entry_special_remove (0, &pfx, nm->fib_src_hi);
 996     }
 997 }
 998
 999 void
1000 nat44_ei_free_outside_address_and_port (nat44_ei_address_t *addresses,
1001                                         u32 thread_index, ip4_address_t *addr,
1002                                         u16 port, nat_protocol_t protocol)
1003 {
1004   nat44_ei_main_t *nm = &nat44_ei_main;
1005   nat44_ei_address_t *a;
1006   u32 address_index;
1007   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1008
1009   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1010     {
1011       if (addresses[address_index].addr.as_u32 == addr->as_u32)
1012         break;
1013     }
1014
1015   ASSERT (address_index < vec_len (addresses));
1016
1017   a = addresses + address_index;
1018
1019   switch (protocol)
1020     {
1021 #define _(N, i, n, s)                                                         \
1022   case NAT_PROTOCOL_##N:                                                      \
1023     ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1);         \
1024     --a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1025     a->busy_##n##_ports--;                                                    \
1026     a->busy_##n##_ports_per_thread[thread_index]--;                           \
1027     break;
1028       foreach_nat_protocol
1029 #undef _
1030         default : nat_elog_info (nm, "unknown protocol");
1031       return;
1032     }
1033 }
1034
1035 void
1036 nat44_ei_free_session_data_v2 (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1037                                u32 thread_index, u8 is_ha)
1038 {
1039   clib_bihash_kv_8_8_t kv;
1040
1041   /* session lookup tables */
1042   init_nat_i2o_k (&kv, s);
1043   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1044     nat_elog_warn (nm, "in2out key del failed");
1045   init_nat_o2i_k (&kv, s);
1046   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1047     nat_elog_warn (nm, "out2in key del failed");
1048
1049   if (!is_ha)
1050     nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1051                              &s->in2out.addr, s->in2out.port, &s->out2in.addr,
1052                              s->out2in.port, s->nat_proto);
1053
1054   if (nat44_ei_is_unk_proto_session (s))
1055     return;
1056
1057   if (!is_ha)
1058     {
1059       /* log NAT event */
1060       nat_ipfix_logging_nat44_ses_delete (
1061         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1062         s->nat_proto, s->in2out.port, s->out2in.port, s->in2out.fib_index);
1063
1064       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1065                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1066                    thread_index);
1067     }
1068
1069   if (nat44_ei_is_session_static (s))
1070     return;
1071
1072   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1073                                           &s->out2in.addr, s->out2in.port,
1074                                           s->nat_proto);
1075 }
1076
1077 nat44_ei_user_t *
1078 nat44_ei_user_get_or_create (nat44_ei_main_t *nm, ip4_address_t *addr,
1079                              u32 fib_index, u32 thread_index)
1080 {
1081   nat44_ei_user_t *u = 0;
1082   nat44_ei_user_key_t user_key;
1083   clib_bihash_kv_8_8_t kv, value;
1084   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1085   dlist_elt_t *per_user_list_head_elt;
1086
1087   user_key.addr.as_u32 = addr->as_u32;
1088   user_key.fib_index = fib_index;
1089   kv.key = user_key.as_u64;
1090
1091   /* Ever heard of the "user" = src ip4 address before? */
1092   if (clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1093     {
1094       if (pool_elts (tnm->users) >= nm->max_users_per_thread)
1095         {
1096           vlib_increment_simple_counter (&nm->user_limit_reached, thread_index,
1097                                          0, 1);
1098           nat_elog_warn (nm, "maximum user limit reached");
1099           return NULL;
1100         }
1101       /* no, make a new one */
1102       pool_get (tnm->users, u);
1103       clib_memset (u, 0, sizeof (*u));
1104
1105       u->addr.as_u32 = addr->as_u32;
1106       u->fib_index = fib_index;
1107
1108       pool_get (tnm->list_pool, per_user_list_head_elt);
1109
1110       u->sessions_per_user_list_head_index =
1111         per_user_list_head_elt - tnm->list_pool;
1112
1113       clib_dlist_init (tnm->list_pool, u->sessions_per_user_list_head_index);
1114
1115       kv.value = u - tnm->users;
1116
1117       /* add user */
1118       if (clib_bihash_add_del_8_8 (&tnm->user_hash, &kv, 1))
1119         {
1120           nat_elog_warn (nm, "user_hash key add failed");
1121           nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1122           return NULL;
1123         }
1124
1125       vlib_set_simple_counter (&nm->total_users, thread_index, 0,
1126                                pool_elts (tnm->users));
1127     }
1128   else
1129     {
1130       u = pool_elt_at_index (tnm->users, value.value);
1131     }
1132
1133   return u;
1134 }
1135
1136 nat44_ei_session_t *
1137 nat44_ei_session_alloc_or_recycle (nat44_ei_main_t *nm, nat44_ei_user_t *u,
1138                                    u32 thread_index, f64 now)
1139 {
1140   nat44_ei_session_t *s;
1141   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1142   u32 oldest_per_user_translation_list_index, session_index;
1143   dlist_elt_t *oldest_per_user_translation_list_elt;
1144   dlist_elt_t *per_user_translation_list_elt;
1145
1146   /* Over quota? Recycle the least recently used translation */
1147   if ((u->nsessions + u->nstaticsessions) >= nm->max_translations_per_user)
1148     {
1149       oldest_per_user_translation_list_index = clib_dlist_remove_head (
1150         tnm->list_pool, u->sessions_per_user_list_head_index);
1151
1152       ASSERT (oldest_per_user_translation_list_index != ~0);
1153
1154       /* Add it back to the end of the LRU list */
1155       clib_dlist_addtail (tnm->list_pool, u->sessions_per_user_list_head_index,
1156                           oldest_per_user_translation_list_index);
1157       /* Get the list element */
1158       oldest_per_user_translation_list_elt = pool_elt_at_index (
1159         tnm->list_pool, oldest_per_user_translation_list_index);
1160
1161       /* Get the session index from the list element */
1162       session_index = oldest_per_user_translation_list_elt->value;
1163
1164       /* Get the session */
1165       s = pool_elt_at_index (tnm->sessions, session_index);
1166
1167       nat44_ei_free_session_data_v2 (nm, s, thread_index, 0);
1168       if (nat44_ei_is_session_static (s))
1169         u->nstaticsessions--;
1170       else
1171         u->nsessions--;
1172       s->flags = 0;
1173       s->total_bytes = 0;
1174       s->total_pkts = 0;
1175       s->state = 0;
1176       s->ext_host_addr.as_u32 = 0;
1177       s->ext_host_port = 0;
1178       s->ext_host_nat_addr.as_u32 = 0;
1179       s->ext_host_nat_port = 0;
1180     }
1181   else
1182     {
1183       pool_get (tnm->sessions, s);
1184       clib_memset (s, 0, sizeof (*s));
1185
1186       /* Create list elts */
1187       pool_get (tnm->list_pool, per_user_translation_list_elt);
1188       clib_dlist_init (tnm->list_pool,
1189                        per_user_translation_list_elt - tnm->list_pool);
1190
1191       per_user_translation_list_elt->value = s - tnm->sessions;
1192       s->per_user_index = per_user_translation_list_elt - tnm->list_pool;
1193       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1194
1195       clib_dlist_addtail (tnm->list_pool, s->per_user_list_head_index,
1196                           per_user_translation_list_elt - tnm->list_pool);
1197
1198       s->user_index = u - tnm->users;
1199       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1200                                pool_elts (tnm->sessions));
1201     }
1202
1203   s->ha_last_refreshed = now;
1204
1205   return s;
1206 }
1207
1208 void
1209 nat44_ei_free_session_data (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1210                             u32 thread_index, u8 is_ha)
1211 {
1212   clib_bihash_kv_8_8_t kv;
1213
1214   init_nat_i2o_k (&kv, s);
1215   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1216     nat_elog_warn (nm, "in2out key del failed");
1217
1218   init_nat_o2i_k (&kv, s);
1219   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1220     nat_elog_warn (nm, "out2in key del failed");
1221
1222   if (!is_ha)
1223     {
1224       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1225                                &s->in2out.addr, s->in2out.port,
1226                                &s->out2in.addr, s->out2in.port, s->nat_proto);
1227
1228       nat_ipfix_logging_nat44_ses_delete (
1229         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1230         s->nat_proto, s->in2out.port, s->out2in.port, s->in2out.fib_index);
1231
1232       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1233                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1234                    thread_index);
1235     }
1236
1237   if (nat44_ei_is_session_static (s))
1238     return;
1239
1240   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1241                                           &s->out2in.addr, s->out2in.port,
1242                                           s->nat_proto);
1243 }
1244
1245 static_always_inline void
1246 nat44_ei_user_del_sessions (nat44_ei_user_t *u, u32 thread_index)
1247 {
1248   dlist_elt_t *elt;
1249   nat44_ei_session_t *s;
1250
1251   nat44_ei_main_t *nm = &nat44_ei_main;
1252   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1253
1254   // get head
1255   elt =
1256     pool_elt_at_index (tnm->list_pool, u->sessions_per_user_list_head_index);
1257   // get first element
1258   elt = pool_elt_at_index (tnm->list_pool, elt->next);
1259
1260   while (elt->value != ~0)
1261     {
1262       s = pool_elt_at_index (tnm->sessions, elt->value);
1263       elt = pool_elt_at_index (tnm->list_pool, elt->next);
1264
1265       nat44_ei_free_session_data (nm, s, thread_index, 0);
1266       nat44_ei_delete_session (nm, s, thread_index);
1267     }
1268 }
1269
1270 int
1271 nat44_ei_user_del (ip4_address_t *addr, u32 fib_index)
1272 {
1273   int rv = 1;
1274
1275   nat44_ei_main_t *nm = &nat44_ei_main;
1276   nat44_ei_main_per_thread_data_t *tnm;
1277
1278   nat44_ei_user_key_t user_key;
1279   clib_bihash_kv_8_8_t kv, value;
1280
1281   user_key.addr.as_u32 = addr->as_u32;
1282   user_key.fib_index = fib_index;
1283   kv.key = user_key.as_u64;
1284
1285   if (nm->num_workers > 1)
1286     {
1287       vec_foreach (tnm, nm->per_thread_data)
1288         {
1289           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1290             {
1291               nat44_ei_user_del_sessions (
1292                 pool_elt_at_index (tnm->users, value.value),
1293                 tnm->thread_index);
1294               rv = 0;
1295               break;
1296             }
1297         }
1298     }
1299   else
1300     {
1301       tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1302       if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1303         {
1304           nat44_ei_user_del_sessions (
1305             pool_elt_at_index (tnm->users, value.value), tnm->thread_index);
1306           rv = 0;
1307         }
1308     }
1309   return rv;
1310 }
1311
1312 void
1313 nat44_ei_static_mapping_del_sessions (nat44_ei_main_t *nm,
1314                                       nat44_ei_main_per_thread_data_t *tnm,
1315                                       nat44_ei_user_key_t u_key, int addr_only,
1316                                       ip4_address_t e_addr, u16 e_port)
1317 {
1318   clib_bihash_kv_8_8_t kv, value;
1319   kv.key = u_key.as_u64;
1320   u64 user_index;
1321   dlist_elt_t *head, *elt;
1322   nat44_ei_user_t *u;
1323   nat44_ei_session_t *s;
1324   u32 elt_index, head_index, ses_index;
1325
1326   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1327     {
1328       user_index = value.value;
1329       u = pool_elt_at_index (tnm->users, user_index);
1330       if (u->nstaticsessions)
1331         {
1332           head_index = u->sessions_per_user_list_head_index;
1333           head = pool_elt_at_index (tnm->list_pool, head_index);
1334           elt_index = head->next;
1335           elt = pool_elt_at_index (tnm->list_pool, elt_index);
1336           ses_index = elt->value;
1337           while (ses_index != ~0)
1338             {
1339               s = pool_elt_at_index (tnm->sessions, ses_index);
1340               elt = pool_elt_at_index (tnm->list_pool, elt->next);
1341               ses_index = elt->value;
1342
1343               if (!addr_only)
1344                 {
1345                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1346                       (s->out2in.port != e_port))
1347                     continue;
1348                 }
1349
1350               if (!nat44_ei_is_session_static (s))
1351                 continue;
1352
1353               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
1354                                              0);
1355               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1356
1357               if (!addr_only)
1358                 break;
1359             }
1360         }
1361     }
1362 }
1363
1364 u32
1365 nat44_ei_get_in2out_worker_index (ip4_header_t *ip0, u32 rx_fib_index0,
1366                                   u8 is_output)
1367 {
1368   nat44_ei_main_t *nm = &nat44_ei_main;
1369   u32 next_worker_index = 0;
1370   u32 hash;
1371
1372   next_worker_index = nm->first_worker_index;
1373   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
1374          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
1375
1376   if (PREDICT_TRUE (is_pow2 (_vec_len (nm->workers))))
1377     next_worker_index += nm->workers[hash & (_vec_len (nm->workers) - 1)];
1378   else
1379     next_worker_index += nm->workers[hash % _vec_len (nm->workers)];
1380
1381   return next_worker_index;
1382 }
1383
1384 u32
1385 nat44_ei_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip0,
1386                                   u32 rx_fib_index0, u8 is_output)
1387 {
1388   nat44_ei_main_t *nm = &nat44_ei_main;
1389   udp_header_t *udp;
1390   u16 port;
1391   clib_bihash_kv_8_8_t kv, value;
1392   nat44_ei_static_mapping_t *m;
1393   u32 proto;
1394   u32 next_worker_index = 0;
1395
1396   /* first try static mappings without port */
1397   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1398     {
1399       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
1400       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1401                                    &value))
1402         {
1403           m = pool_elt_at_index (nm->static_mappings, value.value);
1404           return m->workers[0];
1405         }
1406     }
1407
1408   proto = ip_proto_to_nat_proto (ip0->protocol);
1409   udp = ip4_next_header (ip0);
1410   port = udp->dst_port;
1411
1412   /* unknown protocol */
1413   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
1414     {
1415       /* use current thread */
1416       return vlib_get_thread_index ();
1417     }
1418
1419   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
1420     {
1421       icmp46_header_t *icmp = (icmp46_header_t *) udp;
1422       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
1423       if (!icmp_type_is_error_message (
1424             vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
1425         port = vnet_buffer (b)->ip.reass.l4_src_port;
1426       else
1427         {
1428           /* if error message, then it's not fragmented and we can access it */
1429           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
1430           proto = ip_proto_to_nat_proto (inner_ip->protocol);
1431           void *l4_header = ip4_next_header (inner_ip);
1432           switch (proto)
1433             {
1434             case NAT_PROTOCOL_ICMP:
1435               icmp = (icmp46_header_t *) l4_header;
1436               echo = (icmp_echo_header_t *) (icmp + 1);
1437               port = echo->identifier;
1438               break;
1439             case NAT_PROTOCOL_UDP:
1440             case NAT_PROTOCOL_TCP:
1441               port = ((tcp_udp_header_t *) l4_header)->src_port;
1442               break;
1443             default:
1444               return vlib_get_thread_index ();
1445             }
1446         }
1447     }
1448
1449   /* try static mappings with port */
1450   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1451     {
1452       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
1453       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1454                                    &value))
1455         {
1456           m = pool_elt_at_index (nm->static_mappings, value.value);
1457           return m->workers[0];
1458         }
1459     }
1460
1461   /* worker by outside port */
1462   next_worker_index = nm->first_worker_index;
1463   next_worker_index +=
1464     nm->workers[(clib_net_to_host_u16 (port) - 1024) / nm->port_per_thread];
1465   return next_worker_index;
1466 }
1467
1468 static int
1469 nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
1470                            u32 thread_index, nat_protocol_t proto,
1471                            ip4_address_t *addr, u16 *port, u16 port_per_thread,
1472                            u32 snat_thread_index)
1473 {
1474   nat44_ei_main_t *nm = &nat44_ei_main;
1475   nat44_ei_address_t *a, *ga = 0;
1476   u32 portnum;
1477   int i;
1478
1479   for (i = 0; i < vec_len (addresses); i++)
1480     {
1481       a = addresses + i;
1482       switch (proto)
1483         {
1484 #define _(N, j, n, s)                                                         \
1485   case NAT_PROTOCOL_##N:                                                      \
1486     if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread)       \
1487       {                                                                       \
1488         if (a->fib_index == fib_index)                                        \
1489           {                                                                   \
1490             while (1)                                                         \
1491               {                                                               \
1492                 portnum = (port_per_thread * snat_thread_index) +             \
1493                           nat_random_port (&nm->random_seed, 0,               \
1494                                            port_per_thread - 1) +             \
1495                           1024;                                               \
1496                 if (a->busy_##n##_port_refcounts[portnum])                    \
1497                   continue;                                                   \
1498                 --a->busy_##n##_port_refcounts[portnum];                      \
1499                 a->busy_##n##_ports_per_thread[thread_index]++;               \
1500                 a->busy_##n##_ports++;                                        \
1501                 *addr = a->addr;                                              \
1502                 *port = clib_host_to_net_u16 (portnum);                       \
1503                 return 0;                                                     \
1504               }                                                               \
1505           }                                                                   \
1506         else if (a->fib_index == ~0)                                          \
1507           {                                                                   \
1508             ga = a;                                                           \
1509           }                                                                   \
1510       }                                                                       \
1511     break;
1512           foreach_nat_protocol
1513 #undef _
1514             default : nat_elog_info (nm, "unknown protocol");
1515           return 1;
1516         }
1517     }
1518
1519   if (ga)
1520     {
1521       a = ga;
1522       switch (proto)
1523         {
1524 #define _(N, j, n, s)                                                         \
1525   case NAT_PROTOCOL_##N:                                                      \
1526     while (1)                                                                 \
1527       {                                                                       \
1528         portnum =                                                             \
1529           (port_per_thread * snat_thread_index) +                             \
1530           nat_random_port (&nm->random_seed, 0, port_per_thread - 1) + 1024;  \
1531         if (a->busy_##n##_port_refcounts[portnum])                            \
1532           continue;                                                           \
1533         ++a->busy_##n##_port_refcounts[portnum];                              \
1534         a->busy_##n##_ports_per_thread[thread_index]++;                       \
1535         a->busy_##n##_ports++;                                                \
1536         *addr = a->addr;                                                      \
1537         *port = clib_host_to_net_u16 (portnum);                               \
1538         return 0;                                                             \
1539       }
1540           break;
1541           foreach_nat_protocol
1542 #undef _
1543             default : nat_elog_info (nm, "unknown protocol");
1544           return 1;
1545         }
1546     }
1547
1548   /* Totally out of translations to use... */
1549   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1550   return 1;
1551 }
1552
1553 static int
1554 nat44_ei_alloc_range_cb (nat44_ei_address_t *addresses, u32 fib_index,
1555                          u32 thread_index, nat_protocol_t proto,
1556                          ip4_address_t *addr, u16 *port, u16 port_per_thread,
1557                          u32 snat_thread_index)
1558 {
1559   nat44_ei_main_t *nm = &nat44_ei_main;
1560   nat44_ei_address_t *a = addresses;
1561   u16 portnum, ports;
1562
1563   ports = nm->end_port - nm->start_port + 1;
1564
1565   if (!vec_len (addresses))
1566     goto exhausted;
1567
1568   switch (proto)
1569     {
1570 #define _(N, i, n, s)                                                         \
1571   case NAT_PROTOCOL_##N:                                                      \
1572     if (a->busy_##n##_ports < ports)                                          \
1573       {                                                                       \
1574         while (1)                                                             \
1575           {                                                                   \
1576             portnum = nat_random_port (&nm->random_seed, nm->start_port,      \
1577                                        nm->end_port);                         \
1578             if (a->busy_##n##_port_refcounts[portnum])                        \
1579               continue;                                                       \
1580             ++a->busy_##n##_port_refcounts[portnum];                          \
1581             a->busy_##n##_ports++;                                            \
1582             *addr = a->addr;                                                  \
1583             *port = clib_host_to_net_u16 (portnum);                           \
1584             return 0;                                                         \
1585           }                                                                   \
1586       }                                                                       \
1587     break;
1588       foreach_nat_protocol
1589 #undef _
1590         default : nat_elog_info (nm, "unknown protocol");
1591       return 1;
1592     }
1593
1594 exhausted:
1595   /* Totally out of translations to use... */
1596   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1597   return 1;
1598 }
1599
1600 static int
1601 nat44_ei_alloc_mape_cb (nat44_ei_address_t *addresses, u32 fib_index,
1602                         u32 thread_index, nat_protocol_t proto,
1603                         ip4_address_t *addr, u16 *port, u16 port_per_thread,
1604                         u32 snat_thread_index)
1605 {
1606   nat44_ei_main_t *nm = &nat44_ei_main;
1607   nat44_ei_address_t *a = addresses;
1608   u16 m, ports, portnum, A, j;
1609   m = 16 - (nm->psid_offset + nm->psid_length);
1610   ports = (1 << (16 - nm->psid_length)) - (1 << m);
1611
1612   if (!vec_len (addresses))
1613     goto exhausted;
1614
1615   switch (proto)
1616     {
1617 #define _(N, i, n, s)                                                         \
1618   case NAT_PROTOCOL_##N:                                                      \
1619     if (a->busy_##n##_ports < ports)                                          \
1620       {                                                                       \
1621         while (1)                                                             \
1622           {                                                                   \
1623             A = nat_random_port (&nm->random_seed, 1,                         \
1624                                  pow2_mask (nm->psid_offset));                \
1625             j = nat_random_port (&nm->random_seed, 0, pow2_mask (m));         \
1626             portnum = A | (nm->psid << nm->psid_offset) | (j << (16 - m));    \
1627             if (a->busy_##n##_port_refcounts[portnum])                        \
1628               continue;                                                       \
1629             ++a->busy_##n##_port_refcounts[portnum];                          \
1630             a->busy_##n##_ports++;                                            \
1631             *addr = a->addr;                                                  \
1632             *port = clib_host_to_net_u16 (portnum);                           \
1633             return 0;                                                         \
1634           }                                                                   \
1635       }                                                                       \
1636     break;
1637       foreach_nat_protocol
1638 #undef _
1639         default : nat_elog_info (nm, "unknown protocol");
1640       return 1;
1641     }
1642
1643 exhausted:
1644   /* Totally out of translations to use... */
1645   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1646   return 1;
1647 }
1648
1649 void
1650 nat44_ei_set_alloc_default ()
1651 {
1652   nat44_ei_main_t *nm = &nat44_ei_main;
1653
1654   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
1655   nm->alloc_addr_and_port = nat44_ei_alloc_default_cb;
1656 }
1657
1658 void
1659 nat44_ei_set_alloc_range (u16 start_port, u16 end_port)
1660 {
1661   nat44_ei_main_t *nm = &nat44_ei_main;
1662
1663   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_RANGE;
1664   nm->alloc_addr_and_port = nat44_ei_alloc_range_cb;
1665   nm->start_port = start_port;
1666   nm->end_port = end_port;
1667 }
1668
1669 void
1670 nat44_ei_set_alloc_mape (u16 psid, u16 psid_offset, u16 psid_length)
1671 {
1672   nat44_ei_main_t *nm = &nat44_ei_main;
1673
1674   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_MAPE;
1675   nm->alloc_addr_and_port = nat44_ei_alloc_mape_cb;
1676   nm->psid = psid;
1677   nm->psid_offset = psid_offset;
1678   nm->psid_length = psid_length;
1679 }
1680
1681 static void
1682 nat44_ei_add_static_mapping_when_resolved (ip4_address_t l_addr, u16 l_port,
1683                                            u16 e_port, nat_protocol_t proto,
1684                                            u32 sw_if_index, u32 vrf_id,
1685                                            int addr_only, int identity_nat,
1686                                            u8 *tag)
1687 {
1688   nat44_ei_main_t *nm = &nat44_ei_main;
1689   nat44_ei_static_map_resolve_t *rp;
1690
1691   vec_add2 (nm->to_resolve, rp, 1);
1692   clib_memset (rp, 0, sizeof (*rp));
1693
1694   rp->l_addr.as_u32 = l_addr.as_u32;
1695   rp->l_port = l_port;
1696   rp->e_port = e_port;
1697   rp->sw_if_index = sw_if_index;
1698   rp->vrf_id = vrf_id;
1699   rp->proto = proto;
1700   rp->addr_only = addr_only;
1701   rp->identity_nat = identity_nat;
1702   rp->tag = vec_dup (tag);
1703 }
1704
1705 void
1706 nat44_ei_delete_session (nat44_ei_main_t *nm, nat44_ei_session_t *ses,
1707                          u32 thread_index)
1708 {
1709   nat44_ei_main_per_thread_data_t *tnm =
1710     vec_elt_at_index (nm->per_thread_data, thread_index);
1711   clib_bihash_kv_8_8_t kv, value;
1712   nat44_ei_user_t *u;
1713   const nat44_ei_user_key_t u_key = { .addr = ses->in2out.addr,
1714                                       .fib_index = ses->in2out.fib_index };
1715   const u8 u_static = nat44_ei_is_session_static (ses);
1716
1717   clib_dlist_remove (tnm->list_pool, ses->per_user_index);
1718   pool_put_index (tnm->list_pool, ses->per_user_index);
1719
1720   pool_put (tnm->sessions, ses);
1721   vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1722                            pool_elts (tnm->sessions));
1723
1724   kv.key = u_key.as_u64;
1725   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1726     {
1727       u = pool_elt_at_index (tnm->users, value.value);
1728       if (u_static)
1729         u->nstaticsessions--;
1730       else
1731         u->nsessions--;
1732
1733       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1734     }
1735 }
1736
1737 int
1738 nat44_ei_del_session (nat44_ei_main_t *nm, ip4_address_t *addr, u16 port,
1739                       nat_protocol_t proto, u32 vrf_id, int is_in)
1740 {
1741   nat44_ei_main_per_thread_data_t *tnm;
1742   clib_bihash_kv_8_8_t kv, value;
1743   ip4_header_t ip;
1744   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
1745   nat44_ei_session_t *s;
1746   clib_bihash_8_8_t *t;
1747
1748   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
1749   if (nm->num_workers > 1)
1750     tnm = vec_elt_at_index (nm->per_thread_data,
1751                             nm->worker_in2out_cb (&ip, fib_index, 0));
1752   else
1753     tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1754
1755   init_nat_k (&kv, *addr, port, fib_index, proto);
1756   t = is_in ? &nm->in2out : &nm->out2in;
1757   if (!clib_bihash_search_8_8 (t, &kv, &value))
1758     {
1759       if (pool_is_free_index (tnm->sessions, value.value))
1760         return VNET_API_ERROR_UNSPECIFIED;
1761
1762       s = pool_elt_at_index (tnm->sessions, value.value);
1763       nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data, 0);
1764       nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1765       return 0;
1766     }
1767
1768   return VNET_API_ERROR_NO_SUCH_ENTRY;
1769 }
1770
1771 u32
1772 nat44_ei_get_thread_idx_by_port (u16 e_port)
1773 {
1774   nat44_ei_main_t *nm = &nat44_ei_main;
1775   u32 thread_idx = nm->num_workers;
1776   if (nm->num_workers > 1)
1777     {
1778       thread_idx = nm->first_worker_index +
1779                    nm->workers[(e_port - 1024) / nm->port_per_thread];
1780     }
1781   return thread_idx;
1782 }
1783
1784 void
1785 nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
1786                               int is_add)
1787 {
1788   nat44_ei_main_t *nm = &nat44_ei_main;
1789   fib_prefix_t prefix = {
1790     .fp_len = p_len,
1791     .fp_proto = FIB_PROTOCOL_IP4,
1792     .fp_addr = {
1793                 .ip4.as_u32 = addr->as_u32,
1794                 },
1795   };
1796   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1797
1798   if (is_add)
1799     fib_table_entry_update_one_path (
1800       fib_index, &prefix, nm->fib_src_low,
1801       (FIB_ENTRY_FLAG_CONNECTED | FIB_ENTRY_FLAG_LOCAL |
1802        FIB_ENTRY_FLAG_EXCLUSIVE),
1803       DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
1804   else
1805     fib_table_entry_delete (fib_index, &prefix, nm->fib_src_low);
1806 }
1807
1808 int
1809 nat44_ei_add_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
1810                                  u16 l_port, u16 e_port, nat_protocol_t proto,
1811                                  u32 sw_if_index, u32 vrf_id, u8 addr_only,
1812                                  u8 identity_nat, u8 *tag, u8 is_add)
1813 {
1814   nat44_ei_main_t *nm = &nat44_ei_main;
1815   nat44_ei_static_mapping_t *m = 0;
1816   clib_bihash_kv_8_8_t kv, value;
1817   nat44_ei_address_t *a = 0;
1818   u32 fib_index = ~0;
1819   nat44_ei_interface_t *interface;
1820   nat44_ei_main_per_thread_data_t *tnm;
1821   nat44_ei_user_key_t u_key;
1822   nat44_ei_user_t *u;
1823   dlist_elt_t *head, *elt;
1824   u32 elt_index, head_index;
1825   u32 ses_index;
1826   u64 user_index;
1827   nat44_ei_session_t *s;
1828   nat44_ei_static_map_resolve_t *rp, *rp_match = 0;
1829   nat44_ei_lb_addr_port_t *local;
1830   u32 find = ~0;
1831   int i;
1832
1833   if (sw_if_index != ~0)
1834     {
1835       ip4_address_t *first_int_addr;
1836
1837       for (i = 0; i < vec_len (nm->to_resolve); i++)
1838         {
1839           rp = nm->to_resolve + i;
1840           if (rp->sw_if_index != sw_if_index ||
1841               rp->l_addr.as_u32 != l_addr.as_u32 || rp->vrf_id != vrf_id ||
1842               rp->addr_only != addr_only)
1843             continue;
1844
1845           if (!addr_only)
1846             {
1847               if ((rp->l_port != l_port && rp->e_port != e_port) ||
1848                   rp->proto != proto)
1849                 continue;
1850             }
1851
1852           rp_match = rp;
1853           break;
1854         }
1855
1856       /* Might be already set... */
1857       first_int_addr = ip4_interface_first_address (
1858         nm->ip4_main, sw_if_index, 0 /* just want the address */);
1859
1860       if (is_add)
1861         {
1862           if (rp_match)
1863             return VNET_API_ERROR_VALUE_EXIST;
1864
1865           nat44_ei_add_static_mapping_when_resolved (
1866             l_addr, l_port, e_port, proto, sw_if_index, vrf_id, addr_only,
1867             identity_nat, tag);
1868
1869           /* DHCP resolution required? */
1870           if (!first_int_addr)
1871             return 0;
1872
1873           e_addr.as_u32 = first_int_addr->as_u32;
1874           /* Identity mapping? */
1875           if (l_addr.as_u32 == 0)
1876             l_addr.as_u32 = e_addr.as_u32;
1877         }
1878       else
1879         {
1880           if (!rp_match)
1881             return VNET_API_ERROR_NO_SUCH_ENTRY;
1882
1883           vec_del1 (nm->to_resolve, i);
1884
1885           if (!first_int_addr)
1886             return 0;
1887
1888           e_addr.as_u32 = first_int_addr->as_u32;
1889           /* Identity mapping? */
1890           if (l_addr.as_u32 == 0)
1891             l_addr.as_u32 = e_addr.as_u32;
1892         }
1893     }
1894
1895   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
1896   if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
1897     m = pool_elt_at_index (nm->static_mappings, value.value);
1898
1899   if (is_add)
1900     {
1901       if (m)
1902         {
1903           // identity mapping for second vrf
1904           if (nat44_ei_is_identity_static_mapping (m))
1905             {
1906               pool_foreach (local, m->locals)
1907                 {
1908                   if (local->vrf_id == vrf_id)
1909                     return VNET_API_ERROR_VALUE_EXIST;
1910                 }
1911               pool_get (m->locals, local);
1912               local->vrf_id = vrf_id;
1913               local->fib_index = fib_table_find_or_create_and_lock (
1914                 FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
1915               init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
1916                            m->proto, 0, m - nm->static_mappings);
1917               clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
1918               return 0;
1919             }
1920           return VNET_API_ERROR_VALUE_EXIST;
1921         }
1922
1923       /* Convert VRF id to FIB index */
1924       if (vrf_id != ~0)
1925         {
1926           fib_index = fib_table_find_or_create_and_lock (
1927             FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
1928         }
1929       /* If not specified use inside VRF id from NAT44 plugin config */
1930       else
1931         {
1932           fib_index = nm->inside_fib_index;
1933           vrf_id = nm->inside_vrf_id;
1934           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
1935         }
1936
1937       if (!identity_nat)
1938         {
1939           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
1940                       addr_only ? 0 : proto);
1941           if (!clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
1942                                        &value))
1943             return VNET_API_ERROR_VALUE_EXIST;
1944         }
1945
1946       /* Find external address in allocated addresses and reserve port for
1947          address and port pair mapping when dynamic translations enabled */
1948       if (!(addr_only || nm->static_mapping_only))
1949         {
1950           for (i = 0; i < vec_len (nm->addresses); i++)
1951             {
1952               if (nm->addresses[i].addr.as_u32 == e_addr.as_u32)
1953                 {
1954                   a = nm->addresses + i;
1955                   /* External port must be unused */
1956                   switch (proto)
1957                     {
1958 #define _(N, j, n, s)                                                         \
1959   case NAT_PROTOCOL_##N:                                                      \
1960     if (a->busy_##n##_port_refcounts[e_port])                                 \
1961       return VNET_API_ERROR_INVALID_VALUE;                                    \
1962     ++a->busy_##n##_port_refcounts[e_port];                                   \
1963     if (e_port > 1024)                                                        \
1964       {                                                                       \
1965         a->busy_##n##_ports++;                                                \
1966         a->busy_##n##_ports_per_thread[nat44_ei_get_thread_idx_by_port (      \
1967           e_port)]++;                                                         \
1968       }                                                                       \
1969     break;
1970                       foreach_nat_protocol
1971 #undef _
1972                         default : nat_elog_info (nm, "unknown protocol");
1973                       return VNET_API_ERROR_INVALID_VALUE_2;
1974                     }
1975                   break;
1976                 }
1977             }
1978           /* External address must be allocated */
1979           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1980             {
1981               if (sw_if_index != ~0)
1982                 {
1983                   for (i = 0; i < vec_len (nm->to_resolve); i++)
1984                     {
1985                       rp = nm->to_resolve + i;
1986                       if (rp->addr_only)
1987                         continue;
1988                       if (rp->sw_if_index != sw_if_index &&
1989                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1990                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1991                           rp->e_port != e_port && rp->proto != proto)
1992                         continue;
1993
1994                       vec_del1 (nm->to_resolve, i);
1995                       break;
1996                     }
1997                 }
1998               return VNET_API_ERROR_NO_SUCH_ENTRY;
1999             }
2000         }
2001
2002       pool_get (nm->static_mappings, m);
2003       clib_memset (m, 0, sizeof (*m));
2004       m->tag = vec_dup (tag);
2005       m->local_addr = l_addr;
2006       m->external_addr = e_addr;
2007
2008       if (addr_only)
2009         m->flags |= NAT44_EI_STATIC_MAPPING_FLAG_ADDR_ONLY;
2010       else
2011         {
2012           m->local_port = l_port;
2013           m->external_port = e_port;
2014           m->proto = proto;
2015         }
2016
2017       if (identity_nat)
2018         {
2019           m->flags |= NAT44_EI_STATIC_MAPPING_FLAG_IDENTITY_NAT;
2020           pool_get (m->locals, local);
2021           local->vrf_id = vrf_id;
2022           local->fib_index = fib_index;
2023         }
2024       else
2025         {
2026           m->vrf_id = vrf_id;
2027           m->fib_index = fib_index;
2028         }
2029
2030       if (nm->num_workers > 1)
2031         {
2032           ip4_header_t ip = {
2033             .src_address = m->local_addr,
2034           };
2035           vec_add1 (m->workers, nm->worker_in2out_cb (&ip, m->fib_index, 0));
2036           tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2037         }
2038       else
2039         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2040
2041       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
2042                    m - nm->static_mappings);
2043       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2044
2045       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
2046                    m - nm->static_mappings);
2047       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 1);
2048
2049       /* Delete dynamic sessions matching local address (+ local port) */
2050       // TODO: based on type of NAT EI/ED
2051       if (!(nm->static_mapping_only))
2052         {
2053           u_key.addr = m->local_addr;
2054           u_key.fib_index = m->fib_index;
2055           kv.key = u_key.as_u64;
2056           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2057             {
2058               user_index = value.value;
2059               u = pool_elt_at_index (tnm->users, user_index);
2060               if (u->nsessions)
2061                 {
2062                   head_index = u->sessions_per_user_list_head_index;
2063                   head = pool_elt_at_index (tnm->list_pool, head_index);
2064                   elt_index = head->next;
2065                   elt = pool_elt_at_index (tnm->list_pool, elt_index);
2066                   ses_index = elt->value;
2067                   while (ses_index != ~0)
2068                     {
2069                       s = pool_elt_at_index (tnm->sessions, ses_index);
2070                       elt = pool_elt_at_index (tnm->list_pool, elt->next);
2071                       ses_index = elt->value;
2072
2073                       if (nat44_ei_is_session_static (s))
2074                         continue;
2075
2076                       if (!addr_only && s->in2out.port != m->local_port)
2077                         continue;
2078
2079                       nat44_ei_free_session_data_v2 (
2080                         nm, s, tnm - nm->per_thread_data, 0);
2081                       nat44_ei_delete_session (nm, s,
2082                                                tnm - nm->per_thread_data);
2083
2084                       if (!addr_only)
2085                         break;
2086                     }
2087                 }
2088             }
2089         }
2090     }
2091   else
2092     {
2093       if (!m)
2094         {
2095           if (sw_if_index != ~0)
2096             return 0;
2097           else
2098             return VNET_API_ERROR_NO_SUCH_ENTRY;
2099         }
2100
2101       if (identity_nat)
2102         {
2103           if (vrf_id == ~0)
2104             vrf_id = nm->inside_vrf_id;
2105
2106           pool_foreach (local, m->locals)
2107             {
2108               if (local->vrf_id == vrf_id)
2109                 find = local - m->locals;
2110             }
2111           if (find == ~0)
2112             return VNET_API_ERROR_NO_SUCH_ENTRY;
2113
2114           local = pool_elt_at_index (m->locals, find);
2115           fib_index = local->fib_index;
2116           pool_put (m->locals, local);
2117         }
2118       else
2119         fib_index = m->fib_index;
2120
2121       /* Free external address port */
2122       if (!(addr_only || nm->static_mapping_only))
2123         {
2124           for (i = 0; i < vec_len (nm->addresses); i++)
2125             {
2126               if (nm->addresses[i].addr.as_u32 == e_addr.as_u32)
2127                 {
2128                   a = nm->addresses + i;
2129                   switch (proto)
2130                     {
2131 #define _(N, j, n, s)                                                         \
2132   case NAT_PROTOCOL_##N:                                                      \
2133     --a->busy_##n##_port_refcounts[e_port];                                   \
2134     if (e_port > 1024)                                                        \
2135       {                                                                       \
2136         a->busy_##n##_ports--;                                                \
2137         a->busy_##n##_ports_per_thread[nat44_ei_get_thread_idx_by_port (      \
2138           e_port)]--;                                                         \
2139       }                                                                       \
2140     break;
2141                       foreach_nat_protocol
2142 #undef _
2143                         default : return VNET_API_ERROR_INVALID_VALUE_2;
2144                     }
2145                   break;
2146                 }
2147             }
2148         }
2149
2150       if (nm->num_workers > 1)
2151         tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2152       else
2153         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2154
2155       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
2156       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 0);
2157
2158       /* Delete session(s) for static mapping if exist */
2159       if (!(nm->static_mapping_only) ||
2160           (nm->static_mapping_only && nm->static_mapping_connection_tracking))
2161         {
2162           u_key.addr = m->local_addr;
2163           u_key.fib_index = fib_index;
2164           kv.key = u_key.as_u64;
2165           nat44_ei_static_mapping_del_sessions (nm, tnm, u_key, addr_only,
2166                                                 e_addr, e_port);
2167         }
2168
2169       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2170       if (pool_elts (m->locals))
2171         return 0;
2172
2173       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
2174       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 0);
2175
2176       vec_free (m->tag);
2177       vec_free (m->workers);
2178       /* Delete static mapping from pool */
2179       pool_put (nm->static_mappings, m);
2180     }
2181
2182   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
2183     return 0;
2184
2185   /* Add/delete external address to FIB */
2186   pool_foreach (interface, nm->interfaces)
2187     {
2188       if (nat44_ei_interface_is_inside (interface) || nm->out2in_dpo)
2189         continue;
2190
2191       nat44_ei_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index,
2192                                     is_add);
2193       break;
2194     }
2195   pool_foreach (interface, nm->output_feature_interfaces)
2196     {
2197       if (nat44_ei_interface_is_inside (interface) || nm->out2in_dpo)
2198         continue;
2199
2200       nat44_ei_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index,
2201                                     is_add);
2202       break;
2203     }
2204   return 0;
2205 }
2206
2207 int
2208 nat44_ei_static_mapping_match (ip4_address_t match_addr, u16 match_port,
2209                                u32 match_fib_index,
2210                                nat_protocol_t match_protocol,
2211                                ip4_address_t *mapping_addr, u16 *mapping_port,
2212                                u32 *mapping_fib_index, u8 by_external,
2213                                u8 *is_addr_only, u8 *is_identity_nat)
2214 {
2215   nat44_ei_main_t *nm = &nat44_ei_main;
2216   clib_bihash_kv_8_8_t kv, value;
2217   nat44_ei_static_mapping_t *m;
2218   u16 port;
2219
2220   if (by_external)
2221     {
2222       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2223       if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2224                                   &value))
2225         {
2226           /* Try address only mapping */
2227           init_nat_k (&kv, match_addr, 0, 0, 0);
2228           if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2229                                       &value))
2230             return 1;
2231         }
2232       m = pool_elt_at_index (nm->static_mappings, value.value);
2233
2234       *mapping_fib_index = m->fib_index;
2235       *mapping_addr = m->local_addr;
2236       port = m->local_port;
2237     }
2238   else
2239     {
2240       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2241                   match_protocol);
2242       if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2243         {
2244           /* Try address only mapping */
2245           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2246           if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
2247                                       &value))
2248             return 1;
2249         }
2250       m = pool_elt_at_index (nm->static_mappings, value.value);
2251
2252       *mapping_fib_index = nm->outside_fib_index;
2253       *mapping_addr = m->external_addr;
2254       port = m->external_port;
2255     }
2256
2257   /* Address only mapping doesn't change port */
2258   if (nat44_ei_is_addr_only_static_mapping (m))
2259     *mapping_port = match_port;
2260   else
2261     *mapping_port = port;
2262
2263   if (PREDICT_FALSE (is_addr_only != 0))
2264     *is_addr_only = nat44_ei_is_addr_only_static_mapping (m);
2265
2266   if (PREDICT_FALSE (is_identity_nat != 0))
2267     *is_identity_nat = nat44_ei_is_identity_static_mapping (m);
2268
2269   return 0;
2270 }
2271
2272 static void
2273 nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm)
2274 {
2275   pool_free (tnm->list_pool);
2276   pool_free (tnm->lru_pool);
2277   pool_free (tnm->sessions);
2278   pool_free (tnm->users);
2279
2280   clib_bihash_free_8_8 (&tnm->user_hash);
2281 }
2282
2283 u8 *
2284 format_nat44_ei_key (u8 *s, va_list *args)
2285 {
2286   u64 key = va_arg (*args, u64);
2287
2288   ip4_address_t addr;
2289   u16 port;
2290   nat_protocol_t protocol;
2291   u32 fib_index;
2292
2293   split_nat_key (key, &addr, &port, &fib_index, &protocol);
2294
2295   s = format (s, "%U proto %U port %d fib %d", format_ip4_address, &addr,
2296               format_nat_protocol, protocol, clib_net_to_host_u16 (port),
2297               fib_index);
2298   return s;
2299 }
2300
2301 u8 *
2302 format_nat44_ei_user_kvp (u8 *s, va_list *args)
2303 {
2304   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2305   nat44_ei_user_key_t k;
2306
2307   k.as_u64 = v->key;
2308
2309   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2310               k.fib_index, v->value);
2311
2312   return s;
2313 }
2314
2315 u8 *
2316 format_nat44_ei_session_kvp (u8 *s, va_list *args)
2317 {
2318   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2319
2320   s =
2321     format (s, "%U session-index %llu", format_nat44_ei_key, v->key, v->value);
2322
2323   return s;
2324 }
2325
2326 u8 *
2327 format_nat44_ei_static_mapping_kvp (u8 *s, va_list *args)
2328 {
2329   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2330
2331   s = format (s, "%U static-mapping-index %llu", format_nat44_ei_key, v->key,
2332               v->value);
2333
2334   return s;
2335 }
2336
2337 static void
2338 nat44_ei_worker_db_init (nat44_ei_main_per_thread_data_t *tnm,
2339                          u32 translations, u32 translation_buckets,
2340                          u32 user_buckets)
2341 {
2342   dlist_elt_t *head;
2343
2344   pool_alloc (tnm->list_pool, translations);
2345   pool_alloc (tnm->lru_pool, translations);
2346   pool_alloc (tnm->sessions, translations);
2347
2348   clib_bihash_init_8_8 (&tnm->user_hash, "users", user_buckets, 0);
2349
2350   clib_bihash_set_kvp_format_fn_8_8 (&tnm->user_hash,
2351                                      format_nat44_ei_user_kvp);
2352
2353   pool_get (tnm->lru_pool, head);
2354   tnm->tcp_trans_lru_head_index = head - tnm->lru_pool;
2355   clib_dlist_init (tnm->lru_pool, tnm->tcp_trans_lru_head_index);
2356
2357   pool_get (tnm->lru_pool, head);
2358   tnm->tcp_estab_lru_head_index = head - tnm->lru_pool;
2359   clib_dlist_init (tnm->lru_pool, tnm->tcp_estab_lru_head_index);
2360
2361   pool_get (tnm->lru_pool, head);
2362   tnm->udp_lru_head_index = head - tnm->lru_pool;
2363   clib_dlist_init (tnm->lru_pool, tnm->udp_lru_head_index);
2364
2365   pool_get (tnm->lru_pool, head);
2366   tnm->icmp_lru_head_index = head - tnm->lru_pool;
2367   clib_dlist_init (tnm->lru_pool, tnm->icmp_lru_head_index);
2368
2369   pool_get (tnm->lru_pool, head);
2370   tnm->unk_proto_lru_head_index = head - tnm->lru_pool;
2371   clib_dlist_init (tnm->lru_pool, tnm->unk_proto_lru_head_index);
2372 }
2373
2374 static void
2375 nat44_ei_db_free ()
2376 {
2377   nat44_ei_main_t *nm = &nat44_ei_main;
2378   nat44_ei_main_per_thread_data_t *tnm;
2379
2380   pool_free (nm->static_mappings);
2381   clib_bihash_free_8_8 (&nm->static_mapping_by_local);
2382   clib_bihash_free_8_8 (&nm->static_mapping_by_external);
2383
2384   if (nm->pat)
2385     {
2386       clib_bihash_free_8_8 (&nm->in2out);
2387       clib_bihash_free_8_8 (&nm->out2in);
2388       vec_foreach (tnm, nm->per_thread_data)
2389         {
2390           nat44_ei_worker_db_free (tnm);
2391         }
2392     }
2393 }
2394
2395 static void
2396 nat44_ei_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
2397 {
2398   nat44_ei_main_t *nm = &nat44_ei_main;
2399   nat44_ei_main_per_thread_data_t *tnm;
2400
2401   u32 static_mapping_buckets = 1024;
2402   u32 static_mapping_memory_size = 64 << 20;
2403
2404   clib_bihash_init_8_8 (&nm->static_mapping_by_local,
2405                         "static_mapping_by_local", static_mapping_buckets,
2406                         static_mapping_memory_size);
2407   clib_bihash_init_8_8 (&nm->static_mapping_by_external,
2408                         "static_mapping_by_external", static_mapping_buckets,
2409                         static_mapping_memory_size);
2410   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_local,
2411                                      format_nat44_ei_static_mapping_kvp);
2412   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_external,
2413                                      format_nat44_ei_static_mapping_kvp);
2414
2415   if (nm->pat)
2416     {
2417       clib_bihash_init_8_8 (&nm->in2out, "in2out", translation_buckets, 0);
2418       clib_bihash_init_8_8 (&nm->out2in, "out2in", translation_buckets, 0);
2419       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2420                                          format_nat44_ei_session_kvp);
2421       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2422                                          format_nat44_ei_session_kvp);
2423       vec_foreach (tnm, nm->per_thread_data)
2424         {
2425           nat44_ei_worker_db_init (tnm, translations, translation_buckets,
2426                                    user_buckets);
2427         }
2428     }
2429 }
2430
2431 void
2432 nat44_ei_sessions_clear ()
2433 {
2434   nat44_ei_main_t *nm = &nat44_ei_main;
2435   nat44_ei_main_per_thread_data_t *tnm;
2436
2437   if (nm->pat)
2438     {
2439       clib_bihash_free_8_8 (&nm->in2out);
2440       clib_bihash_free_8_8 (&nm->out2in);
2441       clib_bihash_init_8_8 (&nm->in2out, "in2out", nm->translation_buckets, 0);
2442       clib_bihash_init_8_8 (&nm->out2in, "out2in", nm->translation_buckets, 0);
2443       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2444                                          format_nat44_ei_session_kvp);
2445       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2446                                          format_nat44_ei_session_kvp);
2447       vec_foreach (tnm, nm->per_thread_data)
2448         {
2449           nat44_ei_worker_db_free (tnm);
2450           nat44_ei_worker_db_init (tnm, nm->translations,
2451                                    nm->translation_buckets, nm->user_buckets);
2452         }
2453     }
2454
2455   vlib_zero_simple_counter (&nm->total_users, 0);
2456   vlib_zero_simple_counter (&nm->total_sessions, 0);
2457   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
2458 }
2459
2460 static void
2461 nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque, u32 sw_if_index,
2462                              u32 new_fib_index, u32 old_fib_index)
2463 {
2464   nat44_ei_main_t *nm = &nat44_ei_main;
2465   nat44_ei_outside_fib_t *outside_fib;
2466   nat44_ei_interface_t *i;
2467   u8 is_add = 1;
2468   u8 match = 0;
2469
2470   if (!nm->enabled || (new_fib_index == old_fib_index) ||
2471       (!vec_len (nm->outside_fibs)))
2472     {
2473       return;
2474     }
2475
2476   pool_foreach (i, nm->interfaces)
2477     {
2478       if (i->sw_if_index == sw_if_index)
2479         {
2480           if (!(nat44_ei_interface_is_outside (i)))
2481             return;
2482           match = 1;
2483         }
2484     }
2485
2486   pool_foreach (i, nm->output_feature_interfaces)
2487     {
2488       if (i->sw_if_index == sw_if_index)
2489         {
2490           if (!(nat44_ei_interface_is_outside (i)))
2491             return;
2492           match = 1;
2493         }
2494     }
2495
2496   if (!match)
2497     return;
2498
2499   vec_foreach (outside_fib, nm->outside_fibs)
2500     {
2501       if (outside_fib->fib_index == old_fib_index)
2502         {
2503           outside_fib->refcount--;
2504           if (!outside_fib->refcount)
2505             vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
2506           break;
2507         }
2508     }
2509
2510   vec_foreach (outside_fib, nm->outside_fibs)
2511     {
2512       if (outside_fib->fib_index == new_fib_index)
2513         {
2514           outside_fib->refcount++;
2515           is_add = 0;
2516           break;
2517         }
2518     }
2519
2520   if (is_add)
2521     {
2522       vec_add2 (nm->outside_fibs, outside_fib, 1);
2523       outside_fib->refcount = 1;
2524       outside_fib->fib_index = new_fib_index;
2525     }
2526 }
2527
2528 int
2529 nat44_ei_add_address (nat44_ei_main_t *nm, ip4_address_t *addr, u32 vrf_id)
2530 {
2531   nat44_ei_address_t *ap;
2532   nat44_ei_interface_t *i;
2533   vlib_thread_main_t *tm = vlib_get_thread_main ();
2534
2535   /* Check if address already exists */
2536   vec_foreach (ap, nm->addresses)
2537     {
2538       if (ap->addr.as_u32 == addr->as_u32)
2539         {
2540           nat44_ei_log_err ("address exist");
2541           return VNET_API_ERROR_VALUE_EXIST;
2542         }
2543     }
2544
2545   vec_add2 (nm->addresses, ap, 1);
2546
2547   ap->addr = *addr;
2548   if (vrf_id != ~0)
2549     ap->fib_index = fib_table_find_or_create_and_lock (
2550       FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2551   else
2552     ap->fib_index = ~0;
2553
2554 #define _(N, i, n, s)                                                         \
2555   clib_memset (ap->busy_##n##_port_refcounts, 0,                              \
2556                sizeof (ap->busy_##n##_port_refcounts));                       \
2557   ap->busy_##n##_ports = 0;                                                   \
2558   ap->busy_##n##_ports_per_thread = 0;                                        \
2559   vec_validate_init_empty (ap->busy_##n##_ports_per_thread,                   \
2560                            tm->n_vlib_mains - 1, 0);
2561   foreach_nat_protocol
2562 #undef _
2563
2564     /* Add external address to FIB */
2565     pool_foreach (i, nm->interfaces)
2566   {
2567     if (nat44_ei_interface_is_inside (i) || nm->out2in_dpo)
2568       continue;
2569
2570     nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, 1);
2571     break;
2572   }
2573   pool_foreach (i, nm->output_feature_interfaces)
2574     {
2575       if (nat44_ei_interface_is_inside (i) || nm->out2in_dpo)
2576         continue;
2577
2578       nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, 1);
2579       break;
2580     }
2581
2582   return 0;
2583 }
2584
2585 int
2586 nat44_ei_add_interface_address (nat44_ei_main_t *nm, u32 sw_if_index,
2587                                 int is_del)
2588 {
2589   ip4_main_t *ip4_main = nm->ip4_main;
2590   ip4_address_t *first_int_addr;
2591   nat44_ei_static_map_resolve_t *rp;
2592   u32 *indices_to_delete = 0;
2593   int i, j;
2594   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
2595
2596   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
2597                                                 0 /* just want the address */);
2598
2599   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
2600     {
2601       if (auto_add_sw_if_indices[i] == sw_if_index)
2602         {
2603           if (is_del)
2604             {
2605               /* if have address remove it */
2606               if (first_int_addr)
2607                 (void) nat44_ei_del_address (nm, first_int_addr[0], 1);
2608               else
2609                 {
2610                   for (j = 0; j < vec_len (nm->to_resolve); j++)
2611                     {
2612                       rp = nm->to_resolve + j;
2613                       if (rp->sw_if_index == sw_if_index)
2614                         vec_add1 (indices_to_delete, j);
2615                     }
2616                   if (vec_len (indices_to_delete))
2617                     {
2618                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
2619                         vec_del1 (nm->to_resolve, j);
2620                       vec_free (indices_to_delete);
2621                     }
2622                 }
2623               vec_del1 (nm->auto_add_sw_if_indices, i);
2624             }
2625           else
2626             return VNET_API_ERROR_VALUE_EXIST;
2627
2628           return 0;
2629         }
2630     }
2631
2632   if (is_del)
2633     return VNET_API_ERROR_NO_SUCH_ENTRY;
2634
2635   /* add to the auto-address list */
2636   vec_add1 (nm->auto_add_sw_if_indices, sw_if_index);
2637
2638   /* If the address is already bound - or static - add it now */
2639   if (first_int_addr)
2640     (void) nat44_ei_add_address (nm, first_int_addr, ~0);
2641
2642   return 0;
2643 }
2644
2645 static int
2646 nat44_ei_is_address_used_in_static_mapping (ip4_address_t addr)
2647 {
2648   nat44_ei_main_t *nm = &nat44_ei_main;
2649   nat44_ei_static_mapping_t *m;
2650   pool_foreach (m, nm->static_mappings)
2651     {
2652       if (nat44_ei_is_addr_only_static_mapping (m) ||
2653           nat44_ei_is_identity_static_mapping (m))
2654         continue;
2655       if (m->external_addr.as_u32 == addr.as_u32)
2656         return 1;
2657     }
2658   return 0;
2659 }
2660
2661 int
2662 nat44_ei_del_address (nat44_ei_main_t *nm, ip4_address_t addr, u8 delete_sm)
2663 {
2664   nat44_ei_address_t *a = 0;
2665   nat44_ei_session_t *ses;
2666   u32 *ses_to_be_removed = 0, *ses_index;
2667   nat44_ei_main_per_thread_data_t *tnm;
2668   nat44_ei_interface_t *interface;
2669   nat44_ei_static_mapping_t *m;
2670   int i;
2671
2672   /* Find SNAT address */
2673   for (i = 0; i < vec_len (nm->addresses); i++)
2674     {
2675       if (nm->addresses[i].addr.as_u32 == addr.as_u32)
2676         {
2677           a = nm->addresses + i;
2678           break;
2679         }
2680     }
2681   if (!a)
2682     {
2683       nat44_ei_log_err ("no such address");
2684       return VNET_API_ERROR_NO_SUCH_ENTRY;
2685     }
2686
2687   if (delete_sm)
2688     {
2689       pool_foreach (m, nm->static_mappings)
2690         {
2691           if (m->external_addr.as_u32 == addr.as_u32)
2692             (void) nat44_ei_add_del_static_mapping (
2693               m->local_addr, m->external_addr, m->local_port, m->external_port,
2694               m->proto, ~0 /* sw_if_index */, m->vrf_id,
2695               nat44_ei_is_addr_only_static_mapping (m),
2696               nat44_ei_is_identity_static_mapping (m), m->tag, 0);
2697         }
2698     }
2699   else
2700     {
2701       /* Check if address is used in some static mapping */
2702       if (nat44_ei_is_address_used_in_static_mapping (addr))
2703         {
2704           nat44_ei_log_err ("address used in static mapping");
2705           return VNET_API_ERROR_UNSPECIFIED;
2706         }
2707     }
2708
2709   if (a->fib_index != ~0)
2710     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2711
2712   /* Delete sessions using address */
2713   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
2714     {
2715       vec_foreach (tnm, nm->per_thread_data)
2716         {
2717           pool_foreach (ses, tnm->sessions)
2718             {
2719               if (ses->out2in.addr.as_u32 == addr.as_u32)
2720                 {
2721                   nat44_ei_free_session_data (nm, ses,
2722                                               tnm - nm->per_thread_data, 0);
2723                   vec_add1 (ses_to_be_removed, ses - tnm->sessions);
2724                 }
2725             }
2726           vec_foreach (ses_index, ses_to_be_removed)
2727             {
2728               ses = pool_elt_at_index (tnm->sessions, ses_index[0]);
2729               nat44_ei_delete_session (nm, ses, tnm - nm->per_thread_data);
2730             }
2731           vec_free (ses_to_be_removed);
2732         }
2733     }
2734
2735 #define _(N, i, n, s) vec_free (a->busy_##n##_ports_per_thread);
2736   foreach_nat_protocol
2737 #undef _
2738     vec_del1 (nm->addresses, i);
2739
2740   /* Delete external address from FIB */
2741   pool_foreach (interface, nm->interfaces)
2742     {
2743       if (nat44_ei_interface_is_inside (interface) || nm->out2in_dpo)
2744         continue;
2745       nat44_ei_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
2746       break;
2747     }
2748
2749   pool_foreach (interface, nm->output_feature_interfaces)
2750     {
2751       if (nat44_ei_interface_is_inside (interface) || nm->out2in_dpo)
2752         continue;
2753       nat44_ei_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
2754       break;
2755     }
2756
2757   return 0;
2758 }
2759
2760 static void
2761 nat44_ei_ip4_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
2762                                            u32 sw_if_index,
2763                                            ip4_address_t *address,
2764                                            u32 address_length,
2765                                            u32 if_address_index, u32 is_delete)
2766 {
2767   nat44_ei_main_t *nm = &nat44_ei_main;
2768   nat44_ei_static_map_resolve_t *rp;
2769   ip4_address_t l_addr;
2770   int i, j;
2771   int rv;
2772   nat44_ei_address_t *addresses = nm->addresses;
2773
2774   if (!nm->enabled)
2775     return;
2776
2777   for (i = 0; i < vec_len (nm->auto_add_sw_if_indices); i++)
2778     {
2779       if (sw_if_index == nm->auto_add_sw_if_indices[i])
2780         goto match;
2781     }
2782
2783   return;
2784
2785 match:
2786   if (!is_delete)
2787     {
2788       /* Don't trip over lease renewal, static config */
2789       for (j = 0; j < vec_len (addresses); j++)
2790         if (addresses[j].addr.as_u32 == address->as_u32)
2791           return;
2792
2793       (void) nat44_ei_add_address (nm, address, ~0);
2794       /* Scan static map resolution vector */
2795       for (j = 0; j < vec_len (nm->to_resolve); j++)
2796         {
2797           rp = nm->to_resolve + j;
2798           if (rp->addr_only)
2799             continue;
2800           /* On this interface? */
2801           if (rp->sw_if_index == sw_if_index)
2802             {
2803               /* Indetity mapping? */
2804               if (rp->l_addr.as_u32 == 0)
2805                 l_addr.as_u32 = address[0].as_u32;
2806               else
2807                 l_addr.as_u32 = rp->l_addr.as_u32;
2808               /* Add the static mapping */
2809               rv = nat44_ei_add_del_static_mapping (
2810                 l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
2811                 ~0 /* sw_if_index */, rp->vrf_id, rp->addr_only,
2812                 rp->identity_nat, rp->tag, 1);
2813               if (rv)
2814                 nat_elog_notice_X1 (
2815                   nm, "nat44_ei_add_del_static_mapping returned %d", "i4", rv);
2816             }
2817         }
2818       return;
2819     }
2820   else
2821     {
2822       (void) nat44_ei_del_address (nm, address[0], 1);
2823       return;
2824     }
2825 }
2826
2827 int
2828 nat44_ei_set_frame_queue_nelts (u32 frame_queue_nelts)
2829 {
2830   fail_if_enabled ();
2831   nat44_ei_main_t *nm = &nat44_ei_main;
2832   nm->frame_queue_nelts = frame_queue_nelts;
2833   return 0;
2834 }
2835
2836 static void
2837 nat44_ei_ip4_add_del_addr_only_sm_cb (ip4_main_t *im, uword opaque,
2838                                       u32 sw_if_index, ip4_address_t *address,
2839                                       u32 address_length, u32 if_address_index,
2840                                       u32 is_delete)
2841 {
2842   nat44_ei_main_t *nm = &nat44_ei_main;
2843   nat44_ei_static_map_resolve_t *rp;
2844   nat44_ei_static_mapping_t *m;
2845   clib_bihash_kv_8_8_t kv, value;
2846   int i, rv;
2847   ip4_address_t l_addr;
2848
2849   if (!nm->enabled)
2850     return;
2851
2852   for (i = 0; i < vec_len (nm->to_resolve); i++)
2853     {
2854       rp = nm->to_resolve + i;
2855       if (rp->addr_only == 0)
2856         continue;
2857       if (rp->sw_if_index == sw_if_index)
2858         goto match;
2859     }
2860
2861   return;
2862
2863 match:
2864   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
2865               nm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
2866   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2867     m = 0;
2868   else
2869     m = pool_elt_at_index (nm->static_mappings, value.value);
2870
2871   if (!is_delete)
2872     {
2873       /* Don't trip over lease renewal, static config */
2874       if (m)
2875         return;
2876     }
2877   else
2878     {
2879       if (!m)
2880         return;
2881     }
2882
2883   /* Indetity mapping? */
2884   if (rp->l_addr.as_u32 == 0)
2885     l_addr.as_u32 = address[0].as_u32;
2886   else
2887     l_addr.as_u32 = rp->l_addr.as_u32;
2888   /* Add the static mapping */
2889
2890   rv = nat44_ei_add_del_static_mapping (
2891     l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
2892     ~0 /* sw_if_index */, rp->vrf_id, rp->addr_only, rp->identity_nat, rp->tag,
2893     !is_delete);
2894   if (rv)
2895     nat_elog_notice_X1 (nm, "nat44_ei_add_del_static_mapping returned %d",
2896                         "i4", rv);
2897 }
2898
2899 VLIB_NODE_FN (nat44_ei_classify_node)
2900 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2901 {
2902   u32 n_left_from, *from, *to_next;
2903   nat44_ei_classify_next_t next_index;
2904   nat44_ei_main_t *nm = &nat44_ei_main;
2905   nat44_ei_static_mapping_t *m;
2906   u32 next_in2out = 0, next_out2in = 0;
2907
2908   from = vlib_frame_vector_args (frame);
2909   n_left_from = frame->n_vectors;
2910   next_index = node->cached_next_index;
2911
2912   while (n_left_from > 0)
2913     {
2914       u32 n_left_to_next;
2915
2916       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2917
2918       while (n_left_from > 0 && n_left_to_next > 0)
2919         {
2920           u32 bi0;
2921           vlib_buffer_t *b0;
2922           u32 next0 = NAT44_EI_CLASSIFY_NEXT_IN2OUT;
2923           ip4_header_t *ip0;
2924           nat44_ei_address_t *ap;
2925           clib_bihash_kv_8_8_t kv0, value0;
2926
2927           /* speculatively enqueue b0 to the current next frame */
2928           bi0 = from[0];
2929           to_next[0] = bi0;
2930           from += 1;
2931           to_next += 1;
2932           n_left_from -= 1;
2933           n_left_to_next -= 1;
2934
2935           b0 = vlib_get_buffer (vm, bi0);
2936           ip0 = vlib_buffer_get_current (b0);
2937
2938           vec_foreach (ap, nm->addresses)
2939             {
2940               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
2941                 {
2942                   next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
2943                   goto enqueue0;
2944                 }
2945             }
2946
2947           if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
2948             {
2949               init_nat_k (&kv0, ip0->dst_address, 0, 0, 0);
2950               /* try to classify the fragment based on IP header alone */
2951               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
2952                                            &kv0, &value0))
2953                 {
2954                   m = pool_elt_at_index (nm->static_mappings, value0.value);
2955                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
2956                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
2957                   goto enqueue0;
2958                 }
2959               init_nat_k (&kv0, ip0->dst_address,
2960                           vnet_buffer (b0)->ip.reass.l4_dst_port, 0,
2961                           ip_proto_to_nat_proto (ip0->protocol));
2962               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
2963                                            &kv0, &value0))
2964                 {
2965                   m = pool_elt_at_index (nm->static_mappings, value0.value);
2966                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
2967                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
2968                 }
2969             }
2970
2971         enqueue0:
2972           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
2973                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
2974             {
2975               nat44_ei_classify_trace_t *t =
2976                 vlib_add_trace (vm, node, b0, sizeof (*t));
2977               t->cached = 0;
2978               t->next_in2out = next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
2979             }
2980
2981           next_in2out += next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT;
2982           next_out2in += next0 == NAT44_EI_CLASSIFY_NEXT_OUT2IN;
2983
2984           /* verify speculative enqueue, maybe switch current next frame */
2985           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2986                                            n_left_to_next, bi0, next0);
2987         }
2988
2989       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2990     }
2991
2992   vlib_node_increment_counter (
2993     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_IN2OUT, next_in2out);
2994   vlib_node_increment_counter (
2995     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_OUT2IN, next_out2in);
2996   return frame->n_vectors;
2997 }
2998
2999 VLIB_REGISTER_NODE (nat44_ei_classify_node) = {
3000   .name = "nat44-ei-classify",
3001   .vector_size = sizeof (u32),
3002   .format_trace = format_nat44_ei_classify_trace,
3003   .type = VLIB_NODE_TYPE_INTERNAL,
3004   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3005   .error_strings = nat44_ei_classify_error_strings,
3006   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3007   .next_nodes = {
3008     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out",
3009     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in",
3010     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3011   },
3012 };
3013
3014 /*
3015  * fd.io coding-style-patch-verification: ON
3016  *
3017  * Local Variables:
3018  * eval: (c-set-style "gnu")
3019  * End:
3020  */