src/plugins/nat/nat44-ei/nat44_ei.c

   1 /*
   2  * nat44_ei.c - nat44 endpoint dependent plugin
   3  *
   4  * Copyright (c) 2020 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  * License for the specific language governing permissions and limitations
  15  * under the License.
  16  */
  17
  18 #include <vnet/plugin/plugin.h>
  19 #include <vpp/app/version.h>
  20
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ip/ip4.h>
  24 #include <vnet/ip/ip_table.h>
  25 #include <vnet/ip/reass/ip4_sv_reass.h>
  26 #include <vnet/fib/fib_table.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <vnet/plugin/plugin.h>
  29
  30 // nat lib
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/nat_inlines.h>
  34 #include <nat/lib/ipfix_logging.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_dpo.h>
  37 #include <nat/nat44-ei/nat44_ei_inlines.h>
  38 #include <nat/nat44-ei/nat44_ei.h>
  39
  40 nat44_ei_main_t nat44_ei_main;
  41
  42 extern vlib_node_registration_t nat44_ei_hairpinning_node;
  43 extern vlib_node_registration_t
  44   nat44_ei_in2out_hairpinning_finish_ip4_lookup_node;
  45 extern vlib_node_registration_t
  46   nat44_ei_in2out_hairpinning_finish_interface_output_node;
  47
  48 #define skip_if_disabled()                                                    \
  49   do                                                                          \
  50     {                                                                         \
  51       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  52       if (PREDICT_FALSE (!nm->enabled))                                       \
  53         return;                                                               \
  54     }                                                                         \
  55   while (0)
  56
  57 #define fail_if_enabled()                                                     \
  58   do                                                                          \
  59     {                                                                         \
  60       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  61       if (PREDICT_FALSE (nm->enabled))                                        \
  62         {                                                                     \
  63           nat44_ei_log_err ("plugin enabled");                                \
  64           return 1;                                                           \
  65         }                                                                     \
  66     }                                                                         \
  67   while (0)
  68
  69 #define fail_if_disabled()                                                    \
  70   do                                                                          \
  71     {                                                                         \
  72       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  73       if (PREDICT_FALSE (!nm->enabled))                                       \
  74         {                                                                     \
  75           nat44_ei_log_err ("plugin disabled");                               \
  76           return 1;                                                           \
  77         }                                                                     \
  78     }                                                                         \
  79   while (0)
  80
  81 /* Hook up input features */
  82 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  83   .arc_name = "ip4-unicast",
  84   .node_name = "nat44-ei-classify",
  85   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  86                                "ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-ei-handoff-classify",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  92                                "ip4-sv-reassembly-feature"),
  93 };
  94 VNET_FEATURE_INIT (ip4_nat44_ei_in2out, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-ei-in2out",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  98                                "ip4-sv-reassembly-feature"),
  99 };
 100 VNET_FEATURE_INIT (ip4_nat44_ei_out2in, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-ei-out2in",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 104                                "ip4-sv-reassembly-feature",
 105                                "ip4-dhcp-client-detect"),
 106 };
 107 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output, static) = {
 108   .arc_name = "ip4-output",
 109   .node_name = "nat44-ei-in2out-output",
 110   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 111                                "ip4-sv-reassembly-output-feature"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ei_hairpinning, static) = {
 114   .arc_name = "ip4-local",
 115   .node_name = "nat44-ei-hairpinning",
 116   .runs_before = VNET_FEATURES ("ip4-local-end-of-arc"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_worker_handoff, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-ei-in2out-worker-handoff",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_worker_handoff, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ei-out2in-worker-handoff",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 127                                "ip4-dhcp-client-detect"),
 128 };
 129 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output_worker_handoff, static) = {
 130   .arc_name = "ip4-output",
 131   .node_name = "nat44-ei-in2out-output-worker-handoff",
 132   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 133                                "ip4-sv-reassembly-output-feature"),
 134 };
 135
 136 VLIB_PLUGIN_REGISTER () = {
 137   .version = VPP_BUILD_VER,
 138   .description = "IPv4 Endpoint-Independent NAT (NAT44 EI)",
 139 };
 140
 141 #define foreach_nat44_ei_classify_error                                       \
 142   _ (NEXT_IN2OUT, "next in2out")                                              \
 143   _ (NEXT_OUT2IN, "next out2in")                                              \
 144   _ (FRAG_CACHED, "fragment cached")
 145
 146 typedef enum
 147 {
 148 #define _(sym, str) NAT44_EI_CLASSIFY_ERROR_##sym,
 149   foreach_nat44_ei_classify_error
 150 #undef _
 151     NAT44_EI_CLASSIFY_N_ERROR,
 152 } nat44_ei_classify_error_t;
 153
 154 static char *nat44_ei_classify_error_strings[] = {
 155 #define _(sym, string) string,
 156   foreach_nat44_ei_classify_error
 157 #undef _
 158 };
 159
 160 typedef enum
 161 {
 162   NAT44_EI_CLASSIFY_NEXT_IN2OUT,
 163   NAT44_EI_CLASSIFY_NEXT_OUT2IN,
 164   NAT44_EI_CLASSIFY_NEXT_DROP,
 165   NAT44_EI_CLASSIFY_N_NEXT,
 166 } nat44_ei_classify_next_t;
 167
 168 typedef struct
 169 {
 170   u8 next_in2out;
 171   u8 cached;
 172 } nat44_ei_classify_trace_t;
 173
 174 void nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len,
 175                                    u32 sw_if_index, int is_add);
 176
 177 static void nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm);
 178
 179 static int nat44_ei_add_static_mapping_internal (
 180   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 181   nat_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 182   ip4_address_t pool_addr, u8 *tag);
 183
 184 static int nat44_ei_del_static_mapping_internal (
 185   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 186   nat_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags);
 187
 188 always_inline void
 189 nat44_ei_port_get (nat44_ei_address_t *a, u8 proto, u16 port)
 190 {
 191   nat44_ei_main_t *nm = &nat44_ei_main;
 192
 193 #define _(N, i, n, s)                                                         \
 194   case NAT_PROTOCOL_##N:                                                      \
 195     a->busy_##n##_port_bitmap =                                               \
 196       clib_bitmap_set (a->busy_##n##_port_bitmap, port, 1);                   \
 197     break;
 198
 199   switch (proto)
 200     {
 201       foreach_nat_protocol;
 202     default:
 203       nat_elog_info (nm, "unknown protocol");
 204     }
 205
 206 #undef _
 207 }
 208
 209 always_inline void
 210 nat44_ei_port_put (nat44_ei_address_t *a, u8 proto, u16 port)
 211 {
 212   nat44_ei_main_t *nm = &nat44_ei_main;
 213
 214 #define _(N, i, n, s)                                                         \
 215   case NAT_PROTOCOL_##N:                                                      \
 216     a->busy_##n##_port_bitmap =                                               \
 217       clib_bitmap_set (a->busy_##n##_port_bitmap, port, 0);                   \
 218     break;
 219
 220   switch (proto)
 221     {
 222       foreach_nat_protocol;
 223     default:
 224       nat_elog_info (nm, "unknown protocol");
 225     }
 226
 227 #undef _
 228 }
 229
 230 always_inline bool
 231 nat44_ei_port_is_used (nat44_ei_address_t *a, u8 proto, u16 port)
 232 {
 233   nat44_ei_main_t *nm = &nat44_ei_main;
 234
 235 #define _(N, i, n, s)                                                         \
 236   case NAT_PROTOCOL_##N:                                                      \
 237     return clib_bitmap_get (a->busy_##n##_port_bitmap, port);
 238
 239   switch (proto)
 240     {
 241       foreach_nat_protocol;
 242     default:
 243       nat_elog_info (nm, "unknown protocol");
 244     }
 245
 246 #undef _
 247
 248   return 0;
 249 }
 250
 251 static u8 *
 252 format_nat44_ei_classify_trace (u8 *s, va_list *args)
 253 {
 254   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 255   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 256   nat44_ei_classify_trace_t *t = va_arg (*args, nat44_ei_classify_trace_t *);
 257   char *next;
 258
 259   if (t->cached)
 260     s = format (s, "nat44-ei-classify: fragment cached");
 261   else
 262     {
 263       next = t->next_in2out ? "nat44-ei-in2out" : "nat44-ei-out2in";
 264       s = format (s, "nat44-ei-classify: next %s", next);
 265     }
 266
 267   return s;
 268 }
 269
 270 static void nat44_ei_db_init (u32 translations, u32 translation_buckets,
 271                               u32 user_buckets);
 272
 273 static void nat44_ei_ip4_add_del_interface_address_cb (
 274   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 275   u32 address_length, u32 if_address_index, u32 is_delete);
 276
 277 static void nat44_ei_ip4_add_del_addr_only_sm_cb (
 278   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 279   u32 address_length, u32 if_address_index, u32 is_delete);
 280
 281 static void nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque,
 282                                          u32 sw_if_index, u32 new_fib_index,
 283                                          u32 old_fib_index);
 284
 285 void
 286 nat44_ei_set_node_indexes (nat44_ei_main_t *nm, vlib_main_t *vm)
 287 {
 288   vlib_node_t *node;
 289   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-out2in");
 290   nm->out2in_node_index = node->index;
 291   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out");
 292   nm->in2out_node_index = node->index;
 293   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out-output");
 294   nm->in2out_output_node_index = node->index;
 295 }
 296
 297 int
 298 nat44_ei_set_workers (uword *bitmap)
 299 {
 300   nat44_ei_main_t *nm = &nat44_ei_main;
 301   int i, j = 0;
 302
 303   if (nm->num_workers < 2)
 304     return VNET_API_ERROR_FEATURE_DISABLED;
 305
 306   if (clib_bitmap_last_set (bitmap) >= nm->num_workers)
 307     return VNET_API_ERROR_INVALID_WORKER;
 308
 309   vec_free (nm->workers);
 310   clib_bitmap_foreach (i, bitmap)
 311     {
 312       vec_add1 (nm->workers, i);
 313       nm->per_thread_data[nm->first_worker_index + i].snat_thread_index = j;
 314       nm->per_thread_data[nm->first_worker_index + i].thread_index = i;
 315       j++;
 316     }
 317
 318   nm->port_per_thread = (0xffff - 1024) / _vec_len (nm->workers);
 319
 320   return 0;
 321 }
 322
 323 #define nat_validate_simple_counter(c, i)                                     \
 324   do                                                                          \
 325     {                                                                         \
 326       vlib_validate_simple_counter (&c, i);                                   \
 327       vlib_zero_simple_counter (&c, i);                                       \
 328     }                                                                         \
 329   while (0);
 330
 331 #define nat_init_simple_counter(c, n, sn)                                     \
 332   do                                                                          \
 333     {                                                                         \
 334       c.name = n;                                                             \
 335       c.stat_segment_name = sn;                                               \
 336       nat_validate_simple_counter (c, 0);                                     \
 337     }                                                                         \
 338   while (0);
 339
 340 static_always_inline void
 341 nat_validate_interface_counters (nat44_ei_main_t *nm, u32 sw_if_index)
 342 {
 343 #define _(x)                                                                  \
 344   nat_validate_simple_counter (nm->counters.fastpath.in2out.x, sw_if_index);  \
 345   nat_validate_simple_counter (nm->counters.fastpath.out2in.x, sw_if_index);  \
 346   nat_validate_simple_counter (nm->counters.slowpath.in2out.x, sw_if_index);  \
 347   nat_validate_simple_counter (nm->counters.slowpath.out2in.x, sw_if_index);
 348   foreach_nat_counter;
 349 #undef _
 350   nat_validate_simple_counter (nm->counters.hairpinning, sw_if_index);
 351 }
 352
 353 static void
 354 nat44_ei_add_del_addr_to_fib_foreach_out_if (ip4_address_t *addr, u8 is_add)
 355 {
 356   nat44_ei_main_t *nm = &nat44_ei_main;
 357   nat44_ei_interface_t *i;
 358
 359   pool_foreach (i, nm->interfaces)
 360     {
 361       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 362         {
 363           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 364         }
 365     }
 366   pool_foreach (i, nm->output_feature_interfaces)
 367     {
 368       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 369         {
 370           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 371         }
 372     }
 373 }
 374
 375 static_always_inline void
 376 nat44_ei_add_del_addr_to_fib_foreach_addr (u32 sw_if_index, u8 is_add)
 377 {
 378   nat44_ei_main_t *nm = &nat44_ei_main;
 379   nat44_ei_address_t *ap;
 380
 381   vec_foreach (ap, nm->addresses)
 382     {
 383       nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, is_add);
 384     }
 385 }
 386
 387 static_always_inline void
 388 nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (u32 sw_if_index, u8 is_add)
 389 {
 390   nat44_ei_main_t *nm = &nat44_ei_main;
 391   nat44_ei_static_mapping_t *m;
 392
 393   pool_foreach (m, nm->static_mappings)
 394     {
 395       if (is_sm_addr_only (m->flags) &&
 396           !(m->local_addr.as_u32 == m->external_addr.as_u32))
 397         {
 398           nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 399                                         is_add);
 400         }
 401     }
 402 }
 403
 404 static int
 405 nat44_ei_is_address_used_in_static_mapping (ip4_address_t addr)
 406 {
 407   nat44_ei_main_t *nm = &nat44_ei_main;
 408   nat44_ei_static_mapping_t *m;
 409   pool_foreach (m, nm->static_mappings)
 410     {
 411       if (is_sm_addr_only (m->flags) || is_sm_identity_nat (m->flags))
 412         {
 413           continue;
 414         }
 415       if (m->external_addr.as_u32 == addr.as_u32)
 416         {
 417           return 1;
 418         }
 419     }
 420   return 0;
 421 }
 422
 423 clib_error_t *
 424 nat44_ei_init (vlib_main_t *vm)
 425 {
 426   nat44_ei_main_t *nm = &nat44_ei_main;
 427   vlib_thread_main_t *tm = vlib_get_thread_main ();
 428   vlib_thread_registration_t *tr;
 429   ip4_add_del_interface_address_callback_t cbi = { 0 };
 430   ip4_table_bind_callback_t cbt = { 0 };
 431   u32 i, num_threads = 0;
 432   uword *p, *bitmap = 0;
 433
 434   clib_memset (nm, 0, sizeof (*nm));
 435
 436   // required
 437   nm->vnet_main = vnet_get_main ();
 438   // convenience
 439   nm->ip4_main = &ip4_main;
 440   nm->api_main = vlibapi_get_main ();
 441   nm->ip4_lookup_main = &ip4_main.lookup_main;
 442
 443   // handoff stuff
 444   nm->fq_out2in_index = ~0;
 445   nm->fq_in2out_index = ~0;
 446   nm->fq_in2out_output_index = ~0;
 447
 448   nm->log_level = NAT_LOG_ERROR;
 449
 450   nat44_ei_set_node_indexes (nm, vm);
 451   nm->log_class = vlib_log_register_class ("nat44-ei", 0);
 452
 453   nat_init_simple_counter (nm->total_users, "total-users",
 454                            "/nat44-ei/total-users");
 455   nat_init_simple_counter (nm->total_sessions, "total-sessions",
 456                            "/nat44-ei/total-sessions");
 457   nat_init_simple_counter (nm->user_limit_reached, "user-limit-reached",
 458                            "/nat44-ei/user-limit-reached");
 459
 460 #define _(x)                                                                  \
 461   nat_init_simple_counter (nm->counters.fastpath.in2out.x, #x,                \
 462                            "/nat44-ei/in2out/fastpath/" #x);                  \
 463   nat_init_simple_counter (nm->counters.fastpath.out2in.x, #x,                \
 464                            "/nat44-ei/out2in/fastpath/" #x);                  \
 465   nat_init_simple_counter (nm->counters.slowpath.in2out.x, #x,                \
 466                            "/nat44-ei/in2out/slowpath/" #x);                  \
 467   nat_init_simple_counter (nm->counters.slowpath.out2in.x, #x,                \
 468                            "/nat44-ei/out2in/slowpath/" #x);
 469   foreach_nat_counter;
 470 #undef _
 471   nat_init_simple_counter (nm->counters.hairpinning, "hairpinning",
 472                            "/nat44-ei/hairpinning");
 473
 474   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 475   if (p)
 476     {
 477       tr = (vlib_thread_registration_t *) p[0];
 478       if (tr)
 479         {
 480           nm->num_workers = tr->count;
 481           nm->first_worker_index = tr->first_index;
 482         }
 483     }
 484   num_threads = tm->n_vlib_mains - 1;
 485   nm->port_per_thread = 0xffff - 1024;
 486   vec_validate (nm->per_thread_data, num_threads);
 487
 488   /* Use all available workers by default */
 489   if (nm->num_workers > 1)
 490     {
 491       for (i = 0; i < nm->num_workers; i++)
 492         bitmap = clib_bitmap_set (bitmap, i, 1);
 493       nat44_ei_set_workers (bitmap);
 494       clib_bitmap_free (bitmap);
 495     }
 496   else
 497     {
 498       nm->per_thread_data[0].snat_thread_index = 0;
 499     }
 500
 501   /* callbacks to call when interface address changes. */
 502   cbi.function = nat44_ei_ip4_add_del_interface_address_cb;
 503   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 504   cbi.function = nat44_ei_ip4_add_del_addr_only_sm_cb;
 505   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 506
 507   /* callbacks to call when interface to table biding changes */
 508   cbt.function = nat44_ei_update_outside_fib;
 509   vec_add1 (nm->ip4_main->table_bind_callbacks, cbt);
 510
 511   nm->fib_src_low = fib_source_allocate (
 512     "nat44-ei-low", FIB_SOURCE_PRIORITY_LOW, FIB_SOURCE_BH_SIMPLE);
 513   nm->fib_src_hi = fib_source_allocate ("nat44-ei-hi", FIB_SOURCE_PRIORITY_HI,
 514                                         FIB_SOURCE_BH_SIMPLE);
 515
 516   // used only by out2in-dpo feature
 517   nat_dpo_module_init ();
 518   nat_ha_init (vm, nm->num_workers, num_threads);
 519
 520   nm->hairpinning_fq_index =
 521     vlib_frame_queue_main_init (nat44_ei_hairpinning_node.index, 0);
 522   nm->in2out_hairpinning_finish_ip4_lookup_node_fq_index =
 523     vlib_frame_queue_main_init (
 524       nat44_ei_in2out_hairpinning_finish_ip4_lookup_node.index, 0);
 525   nm->in2out_hairpinning_finish_interface_output_node_fq_index =
 526     vlib_frame_queue_main_init (
 527       nat44_ei_in2out_hairpinning_finish_interface_output_node.index, 0);
 528   return nat44_ei_api_hookup (vm);
 529 }
 530
 531 VLIB_INIT_FUNCTION (nat44_ei_init);
 532
 533 int
 534 nat44_ei_plugin_enable (nat44_ei_config_t c)
 535 {
 536   nat44_ei_main_t *nm = &nat44_ei_main;
 537
 538   fail_if_enabled ();
 539
 540   if (!c.users)
 541     c.users = 1024;
 542
 543   if (!c.sessions)
 544     c.sessions = 10 * 1024;
 545
 546   if (!c.user_sessions)
 547     c.user_sessions = c.sessions;
 548
 549   nm->rconfig = c;
 550
 551   if (!nm->frame_queue_nelts)
 552     nm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
 553
 554   nm->translations = c.sessions;
 555   nm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
 556   nm->user_buckets = nat_calc_bihash_buckets (c.users);
 557
 558   nm->pat = (!c.static_mapping_only ||
 559              (c.static_mapping_only && c.connection_tracking));
 560
 561   nm->static_mapping_only = c.static_mapping_only;
 562   nm->static_mapping_connection_tracking = c.connection_tracking;
 563   nm->out2in_dpo = c.out2in_dpo;
 564   nm->forwarding_enabled = 0;
 565   nm->mss_clamping = 0;
 566
 567   nm->max_users_per_thread = c.users;
 568   nm->max_translations_per_thread = c.sessions;
 569   nm->max_translations_per_user = c.user_sessions;
 570
 571   nm->inside_vrf_id = c.inside_vrf;
 572   nm->inside_fib_index = fib_table_find_or_create_and_lock (
 573     FIB_PROTOCOL_IP4, c.inside_vrf, nm->fib_src_hi);
 574
 575   nm->outside_vrf_id = c.outside_vrf;
 576   nm->outside_fib_index = fib_table_find_or_create_and_lock (
 577     FIB_PROTOCOL_IP4, c.outside_vrf, nm->fib_src_hi);
 578
 579   nat_reset_timeouts (&nm->timeouts);
 580   nat44_ei_db_init (nm->translations, nm->translation_buckets,
 581                     nm->user_buckets);
 582   nat44_ei_set_alloc_default ();
 583
 584   vlib_zero_simple_counter (&nm->total_users, 0);
 585   vlib_zero_simple_counter (&nm->total_sessions, 0);
 586   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
 587
 588   if (nm->num_workers > 1)
 589     {
 590       if (nm->fq_in2out_index == ~0)
 591         {
 592           nm->fq_in2out_index = vlib_frame_queue_main_init (
 593             nm->in2out_node_index, nm->frame_queue_nelts);
 594         }
 595       if (nm->fq_out2in_index == ~0)
 596         {
 597           nm->fq_out2in_index = vlib_frame_queue_main_init (
 598             nm->out2in_node_index, nm->frame_queue_nelts);
 599         }
 600       if (nm->fq_in2out_output_index == ~0)
 601         {
 602           nm->fq_in2out_output_index = vlib_frame_queue_main_init (
 603             nm->in2out_output_node_index, nm->frame_queue_nelts);
 604         }
 605     }
 606
 607   nat_ha_enable ();
 608   nm->enabled = 1;
 609
 610   return 0;
 611 }
 612
 613 static_always_inline nat44_ei_outside_fib_t *
 614 nat44_ei_get_outside_fib (nat44_ei_outside_fib_t *outside_fibs, u32 fib_index)
 615 {
 616   nat44_ei_outside_fib_t *f;
 617   vec_foreach (f, outside_fibs)
 618     {
 619       if (f->fib_index == fib_index)
 620         {
 621           return f;
 622         }
 623     }
 624   return 0;
 625 }
 626
 627 static_always_inline nat44_ei_interface_t *
 628 nat44_ei_get_interface (nat44_ei_interface_t *interfaces, u32 sw_if_index)
 629 {
 630   nat44_ei_interface_t *i;
 631   pool_foreach (i, interfaces)
 632     {
 633       if (i->sw_if_index == sw_if_index)
 634         {
 635           return i;
 636         }
 637     }
 638   return 0;
 639 }
 640
 641 static_always_inline int
 642 nat44_ei_hairpinning_enable (u8 is_enable)
 643 {
 644   nat44_ei_main_t *nm = &nat44_ei_main;
 645   u32 sw_if_index = 0; // local0
 646
 647   if (is_enable)
 648     {
 649       nm->hairpin_reg += 1;
 650       if (1 == nm->hairpin_reg)
 651         {
 652           return vnet_feature_enable_disable (
 653             "ip4-local", "nat44-ei-hairpinning", sw_if_index, is_enable, 0, 0);
 654         }
 655     }
 656   else
 657     {
 658       if (0 == nm->hairpin_reg)
 659         return 1;
 660
 661       nm->hairpin_reg -= 1;
 662       if (0 == nm->hairpin_reg)
 663         {
 664           return vnet_feature_enable_disable (
 665             "ip4-local", "nat44-ei-hairpinning", sw_if_index, is_enable, 0, 0);
 666         }
 667     }
 668
 669   return 0;
 670 }
 671
 672 int
 673 nat44_ei_add_interface (u32 sw_if_index, u8 is_inside)
 674 {
 675   const char *feature_name, *del_feature_name;
 676   nat44_ei_main_t *nm = &nat44_ei_main;
 677
 678   nat44_ei_outside_fib_t *outside_fib;
 679   nat44_ei_interface_t *i;
 680   u32 fib_index;
 681   int rv;
 682
 683   fail_if_disabled ();
 684
 685   if (nm->out2in_dpo && !is_inside)
 686     {
 687       nat44_ei_log_err ("error unsupported");
 688       return VNET_API_ERROR_UNSUPPORTED;
 689     }
 690
 691   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 692     {
 693       nat44_ei_log_err ("error interface already configured");
 694       return VNET_API_ERROR_VALUE_EXIST;
 695     }
 696
 697   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 698   if (i)
 699     {
 700       if ((nat44_ei_interface_is_inside (i) && is_inside) ||
 701           (nat44_ei_interface_is_outside (i) && !is_inside))
 702         {
 703           return 0;
 704         }
 705       if (nm->num_workers > 1)
 706         {
 707           del_feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 708                                           "nat44-ei-out2in-worker-handoff";
 709           feature_name = "nat44-ei-handoff-classify";
 710         }
 711       else
 712         {
 713           del_feature_name =
 714             !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 715
 716           feature_name = "nat44-ei-classify";
 717         }
 718
 719       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 720       if (rv)
 721         {
 722           return rv;
 723         }
 724       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 725                                         sw_if_index, 0, 0, 0);
 726       if (rv)
 727         {
 728           return rv;
 729         }
 730       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 731                                         sw_if_index, 1, 0, 0);
 732       if (rv)
 733         {
 734           return rv;
 735         }
 736       if (!is_inside)
 737         {
 738           rv = nat44_ei_hairpinning_enable (0);
 739           if (rv)
 740             {
 741               return rv;
 742             }
 743         }
 744     }
 745   else
 746     {
 747       if (nm->num_workers > 1)
 748         {
 749           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 750                                      "nat44-ei-out2in-worker-handoff";
 751         }
 752       else
 753         {
 754           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 755         }
 756       nat_validate_interface_counters (nm, sw_if_index);
 757
 758       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 759       if (rv)
 760         {
 761           return rv;
 762         }
 763       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 764                                         sw_if_index, 1, 0, 0);
 765       if (rv)
 766         {
 767           return rv;
 768         }
 769       if (is_inside && !nm->out2in_dpo)
 770         {
 771           rv = nat44_ei_hairpinning_enable (1);
 772           if (rv)
 773             {
 774               return rv;
 775             }
 776         }
 777
 778       pool_get (nm->interfaces, i);
 779       i->sw_if_index = sw_if_index;
 780       i->flags = 0;
 781     }
 782
 783   fib_index =
 784     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 785
 786   if (!is_inside)
 787     {
 788       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 789
 790       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 791       if (outside_fib)
 792         {
 793           outside_fib->refcount++;
 794         }
 795       else
 796         {
 797           vec_add2 (nm->outside_fibs, outside_fib, 1);
 798           outside_fib->fib_index = fib_index;
 799           outside_fib->refcount = 1;
 800         }
 801
 802       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 803       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 804     }
 805   else
 806     {
 807       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 808     }
 809
 810   return 0;
 811 }
 812
 813 int
 814 nat44_ei_del_interface (u32 sw_if_index, u8 is_inside)
 815 {
 816   const char *feature_name, *del_feature_name;
 817   nat44_ei_main_t *nm = &nat44_ei_main;
 818
 819   nat44_ei_outside_fib_t *outside_fib;
 820   nat44_ei_interface_t *i;
 821   u32 fib_index;
 822   int rv;
 823
 824   fail_if_disabled ();
 825
 826   if (nm->out2in_dpo && !is_inside)
 827     {
 828       nat44_ei_log_err ("error unsupported");
 829       return VNET_API_ERROR_UNSUPPORTED;
 830     }
 831
 832   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 833   if (i == 0)
 834     {
 835       nat44_ei_log_err ("error interface couldn't be found");
 836       return VNET_API_ERROR_NO_SUCH_ENTRY;
 837     }
 838
 839   if (nat44_ei_interface_is_inside (i) && nat44_ei_interface_is_outside (i))
 840     {
 841       if (nm->num_workers > 1)
 842         {
 843           del_feature_name = "nat44-ei-handoff-classify";
 844           feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 845                                       "nat44-ei-out2in-worker-handoff";
 846         }
 847       else
 848         {
 849           del_feature_name = "nat44-ei-classify";
 850           feature_name = !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 851         }
 852
 853       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 854       if (rv)
 855         {
 856           return rv;
 857         }
 858       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 859                                         sw_if_index, 0, 0, 0);
 860       if (rv)
 861         {
 862           return rv;
 863         }
 864       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 865                                         sw_if_index, 1, 0, 0);
 866       if (rv)
 867         {
 868           return rv;
 869         }
 870       if (is_inside)
 871         {
 872           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 873         }
 874       else
 875         {
 876           rv = nat44_ei_hairpinning_enable (1);
 877           if (rv)
 878             {
 879               return rv;
 880             }
 881           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 882         }
 883     }
 884   else
 885     {
 886       if (nm->num_workers > 1)
 887         {
 888           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 889                                      "nat44-ei-out2in-worker-handoff";
 890         }
 891       else
 892         {
 893           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 894         }
 895
 896       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 897       if (rv)
 898         {
 899           return rv;
 900         }
 901       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 902                                         sw_if_index, 0, 0, 0);
 903       if (rv)
 904         {
 905           return rv;
 906         }
 907       if (is_inside)
 908         {
 909           rv = nat44_ei_hairpinning_enable (0);
 910           if (rv)
 911             {
 912               return rv;
 913             }
 914         }
 915
 916       // remove interface
 917       pool_put (nm->interfaces, i);
 918     }
 919
 920   if (!is_inside)
 921     {
 922       fib_index =
 923         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 924       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 925       if (outside_fib)
 926         {
 927           outside_fib->refcount--;
 928           if (!outside_fib->refcount)
 929             {
 930               vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
 931             }
 932         }
 933
 934       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
 935       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
 936     }
 937
 938   return 0;
 939 }
 940
 941 int
 942 nat44_ei_add_output_interface (u32 sw_if_index)
 943 {
 944   nat44_ei_main_t *nm = &nat44_ei_main;
 945
 946   nat44_ei_outside_fib_t *outside_fib;
 947   nat44_ei_interface_t *i;
 948   u32 fib_index;
 949   int rv;
 950
 951   fail_if_disabled ();
 952
 953   if (nat44_ei_get_interface (nm->interfaces, sw_if_index))
 954     {
 955       nat44_ei_log_err ("error interface already configured");
 956       return VNET_API_ERROR_VALUE_EXIST;
 957     }
 958
 959   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 960     {
 961       nat44_ei_log_err ("error interface already configured");
 962       return VNET_API_ERROR_VALUE_EXIST;
 963     }
 964
 965   if (nm->num_workers > 1)
 966     {
 967       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 968       if (rv)
 969         {
 970           return rv;
 971         }
 972       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 973       if (rv)
 974         {
 975           return rv;
 976         }
 977       rv = vnet_feature_enable_disable (
 978         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 1, 0, 0);
 979       if (rv)
 980         {
 981           return rv;
 982         }
 983       rv = vnet_feature_enable_disable (
 984         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 1,
 985         0, 0);
 986       if (rv)
 987         {
 988           return rv;
 989         }
 990     }
 991   else
 992     {
 993       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 994       if (rv)
 995         {
 996           return rv;
 997         }
 998       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 999       if (rv)
1000         {
1001           return rv;
1002         }
1003       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
1004                                         sw_if_index, 1, 0, 0);
1005       if (rv)
1006         {
1007           return rv;
1008         }
1009       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
1010                                         sw_if_index, 1, 0, 0);
1011       if (rv)
1012         {
1013           return rv;
1014         }
1015     }
1016
1017   nat_validate_interface_counters (nm, sw_if_index);
1018
1019   pool_get (nm->output_feature_interfaces, i);
1020   i->sw_if_index = sw_if_index;
1021   i->flags = 0;
1022   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
1023   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
1024
1025   fib_index =
1026     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1027   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
1028   if (outside_fib)
1029     {
1030       outside_fib->refcount++;
1031     }
1032   else
1033     {
1034       vec_add2 (nm->outside_fibs, outside_fib, 1);
1035       outside_fib->fib_index = fib_index;
1036       outside_fib->refcount = 1;
1037     }
1038
1039   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
1040   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
1041
1042   return 0;
1043 }
1044
1045 int
1046 nat44_ei_del_output_interface (u32 sw_if_index)
1047 {
1048   nat44_ei_main_t *nm = &nat44_ei_main;
1049
1050   nat44_ei_outside_fib_t *outside_fib;
1051   nat44_ei_interface_t *i;
1052   u32 fib_index;
1053   int rv;
1054
1055   fail_if_disabled ();
1056
1057   i = nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index);
1058   if (!i)
1059     {
1060       nat44_ei_log_err ("error interface couldn't be found");
1061       return VNET_API_ERROR_NO_SUCH_ENTRY;
1062     }
1063
1064   if (nm->num_workers > 1)
1065     {
1066       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1067       if (rv)
1068         {
1069           return rv;
1070         }
1071       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1072       if (rv)
1073         {
1074           return rv;
1075         }
1076       rv = vnet_feature_enable_disable (
1077         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 0, 0, 0);
1078       if (rv)
1079         {
1080           return rv;
1081         }
1082       rv = vnet_feature_enable_disable (
1083         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 0,
1084         0, 0);
1085       if (rv)
1086         {
1087           return rv;
1088         }
1089     }
1090   else
1091     {
1092       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1093       if (rv)
1094         {
1095           return rv;
1096         }
1097       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1098       if (rv)
1099         {
1100           return rv;
1101         }
1102       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
1103                                         sw_if_index, 0, 0, 0);
1104       if (rv)
1105         {
1106           return rv;
1107         }
1108       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
1109                                         sw_if_index, 0, 0, 0);
1110       if (rv)
1111         {
1112           return rv;
1113         }
1114     }
1115
1116   pool_put (nm->output_feature_interfaces, i);
1117
1118   fib_index =
1119     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1120   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
1121   if (outside_fib)
1122     {
1123       outside_fib->refcount--;
1124       if (!outside_fib->refcount)
1125         {
1126           vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
1127         }
1128     }
1129
1130   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
1131   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
1132
1133   return 0;
1134 }
1135
1136 int
1137 nat44_ei_add_del_output_interface (u32 sw_if_index, int is_del)
1138 {
1139   if (is_del)
1140     {
1141       return nat44_ei_del_output_interface (sw_if_index);
1142     }
1143   else
1144     {
1145       return nat44_ei_add_output_interface (sw_if_index);
1146     }
1147 }
1148
1149 int
1150 nat44_ei_del_addresses ()
1151 {
1152   nat44_ei_main_t *nm = &nat44_ei_main;
1153   nat44_ei_address_t *a, *vec;
1154   int error = 0;
1155
1156   vec = vec_dup (nm->addresses);
1157   vec_foreach (a, vec)
1158     {
1159       error = nat44_ei_del_address (a->addr, 0);
1160
1161       if (error)
1162         {
1163           nat44_ei_log_err ("error occurred while removing adderess");
1164         }
1165     }
1166   vec_free (vec);
1167   vec_free (nm->addresses);
1168   nm->addresses = 0;
1169
1170   vec_free (nm->auto_add_sw_if_indices);
1171   nm->auto_add_sw_if_indices = 0;
1172   return error;
1173 }
1174
1175 int
1176 nat44_ei_del_interfaces ()
1177 {
1178   nat44_ei_main_t *nm = &nat44_ei_main;
1179   nat44_ei_interface_t *i, *pool;
1180   int error = 0;
1181
1182   pool = pool_dup (nm->interfaces);
1183   pool_foreach (i, pool)
1184     {
1185       if (nat44_ei_interface_is_inside (i))
1186         {
1187           error = nat44_ei_del_interface (i->sw_if_index, 1);
1188         }
1189       if (nat44_ei_interface_is_outside (i))
1190         {
1191           error = nat44_ei_del_interface (i->sw_if_index, 0);
1192         }
1193
1194       if (error)
1195         {
1196           nat44_ei_log_err ("error occurred while removing interface");
1197         }
1198     }
1199   pool_free (pool);
1200   pool_free (nm->interfaces);
1201   nm->interfaces = 0;
1202   return error;
1203 }
1204
1205 int
1206 nat44_ei_del_output_interfaces ()
1207 {
1208   nat44_ei_main_t *nm = &nat44_ei_main;
1209   nat44_ei_interface_t *i, *pool;
1210   int error = 0;
1211
1212   pool = pool_dup (nm->output_feature_interfaces);
1213   pool_foreach (i, pool)
1214     {
1215       error = nat44_ei_del_output_interface (i->sw_if_index);
1216       if (error)
1217         {
1218           nat44_ei_log_err ("error occurred while removing output interface");
1219         }
1220     }
1221   pool_free (pool);
1222   pool_free (nm->output_feature_interfaces);
1223   nm->output_feature_interfaces = 0;
1224   return error;
1225 }
1226
1227 int
1228 nat44_ei_del_static_mappings ()
1229 {
1230   nat44_ei_main_t *nm = &nat44_ei_main;
1231   nat44_ei_static_mapping_t *m, *pool;
1232   int error = 0;
1233
1234   pool = pool_dup (nm->static_mappings);
1235   pool_foreach (m, pool)
1236     {
1237       error = nat44_ei_del_static_mapping_internal (
1238         m->local_addr, m->external_addr, m->local_port, m->external_port,
1239         m->proto, m->vrf_id, ~0, m->flags);
1240       if (error)
1241         {
1242           nat44_ei_log_err ("error occurred while removing mapping");
1243         }
1244     }
1245   pool_free (pool);
1246   pool_free (nm->static_mappings);
1247   nm->static_mappings = 0;
1248
1249   vec_free (nm->to_resolve);
1250   nm->to_resolve = 0;
1251
1252   clib_bihash_free_8_8 (&nm->static_mapping_by_local);
1253   clib_bihash_free_8_8 (&nm->static_mapping_by_external);
1254
1255   return error;
1256 }
1257
1258 int
1259 nat44_ei_plugin_disable ()
1260 {
1261   nat44_ei_main_t *nm = &nat44_ei_main;
1262   nat44_ei_main_per_thread_data_t *tnm;
1263   int rc, error = 0;
1264
1265   nat_ha_disable ();
1266
1267   rc = nat44_ei_del_static_mappings ();
1268   if (rc)
1269     error = 1;
1270
1271   rc = nat44_ei_del_addresses ();
1272   if (rc)
1273     error = 1;
1274
1275   rc = nat44_ei_del_interfaces ();
1276   if (rc)
1277     error = 1;
1278
1279   rc = nat44_ei_del_output_interfaces ();
1280   if (rc)
1281     error = 1;
1282
1283   if (nm->pat)
1284     {
1285       clib_bihash_free_8_8 (&nm->in2out);
1286       clib_bihash_free_8_8 (&nm->out2in);
1287
1288       vec_foreach (tnm, nm->per_thread_data)
1289         {
1290           nat44_ei_worker_db_free (tnm);
1291         }
1292     }
1293
1294   clib_memset (&nm->rconfig, 0, sizeof (nm->rconfig));
1295
1296   nm->forwarding_enabled = 0;
1297   nm->enabled = 0;
1298
1299   return error;
1300 }
1301
1302 int
1303 nat44_ei_set_outside_address_and_port (nat44_ei_address_t *addresses,
1304                                        u32 thread_index, ip4_address_t addr,
1305                                        u16 port, nat_protocol_t protocol)
1306 {
1307   nat44_ei_main_t *nm = &nat44_ei_main;
1308   nat44_ei_address_t *a = 0;
1309   u32 address_index;
1310   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1311
1312   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1313     {
1314       if (addresses[address_index].addr.as_u32 != addr.as_u32)
1315         continue;
1316
1317       a = addresses + address_index;
1318       switch (protocol)
1319         {
1320 #define _(N, j, n, s)                                                         \
1321   case NAT_PROTOCOL_##N:                                                      \
1322     if (nat44_ei_port_is_used (a, NAT_PROTOCOL_##N, port_host_byte_order))    \
1323       return VNET_API_ERROR_INSTANCE_IN_USE;                                  \
1324     nat44_ei_port_get (a, NAT_PROTOCOL_##N, port_host_byte_order);            \
1325     a->busy_##n##_ports_per_thread[thread_index]++;                           \
1326     a->busy_##n##_ports++;                                                    \
1327     return 0;
1328           foreach_nat_protocol
1329 #undef _
1330             default : nat_elog_info (nm, "unknown protocol");
1331           return 1;
1332         }
1333     }
1334
1335   return VNET_API_ERROR_NO_SUCH_ENTRY;
1336 }
1337
1338 void
1339 nat44_ei_add_del_address_dpo (ip4_address_t addr, u8 is_add)
1340 {
1341   nat44_ei_main_t *nm = &nat44_ei_main;
1342   dpo_id_t dpo_v4 = DPO_INVALID;
1343   fib_prefix_t pfx = {
1344     .fp_proto = FIB_PROTOCOL_IP4,
1345     .fp_len = 32,
1346     .fp_addr.ip4.as_u32 = addr.as_u32,
1347   };
1348
1349   if (is_add)
1350     {
1351       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
1352       fib_table_entry_special_dpo_add (0, &pfx, nm->fib_src_hi,
1353                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
1354       dpo_reset (&dpo_v4);
1355     }
1356   else
1357     {
1358       fib_table_entry_special_remove (0, &pfx, nm->fib_src_hi);
1359     }
1360 }
1361
1362 void
1363 nat44_ei_free_outside_address_and_port (nat44_ei_address_t *addresses,
1364                                         u32 thread_index, ip4_address_t *addr,
1365                                         u16 port, nat_protocol_t protocol)
1366 {
1367   nat44_ei_main_t *nm = &nat44_ei_main;
1368   nat44_ei_address_t *a;
1369   u32 address_index;
1370   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1371
1372   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1373     {
1374       if (addresses[address_index].addr.as_u32 == addr->as_u32)
1375         break;
1376     }
1377
1378   ASSERT (address_index < vec_len (addresses));
1379
1380   a = addresses + address_index;
1381
1382   switch (protocol)
1383     {
1384 #define _(N, i, n, s)                                                         \
1385   case NAT_PROTOCOL_##N:                                                      \
1386     ASSERT (                                                                  \
1387       nat44_ei_port_is_used (a, NAT_PROTOCOL_##N, port_host_byte_order));     \
1388     nat44_ei_port_put (a, NAT_PROTOCOL_##N, port_host_byte_order);            \
1389     a->busy_##n##_ports--;                                                    \
1390     a->busy_##n##_ports_per_thread[thread_index]--;                           \
1391     break;
1392       foreach_nat_protocol
1393 #undef _
1394         default : nat_elog_info (nm, "unknown protocol");
1395       return;
1396     }
1397 }
1398
1399 void
1400 nat44_ei_free_session_data_v2 (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1401                                u32 thread_index, u8 is_ha)
1402 {
1403   clib_bihash_kv_8_8_t kv;
1404
1405   /* session lookup tables */
1406   init_nat_i2o_k (&kv, s);
1407   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1408     nat_elog_warn (nm, "in2out key del failed");
1409   init_nat_o2i_k (&kv, s);
1410   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1411     nat_elog_warn (nm, "out2in key del failed");
1412
1413   if (!is_ha)
1414     nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1415                              &s->in2out.addr, s->in2out.port, &s->out2in.addr,
1416                              s->out2in.port, s->nat_proto);
1417
1418   if (nat44_ei_is_unk_proto_session (s))
1419     return;
1420
1421   if (!is_ha)
1422     {
1423       /* log NAT event */
1424       nat_ipfix_logging_nat44_ses_delete (
1425         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1426         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1427         s->in2out.fib_index);
1428
1429       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1430                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1431                    thread_index);
1432     }
1433
1434   if (nat44_ei_is_session_static (s))
1435     return;
1436
1437   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1438                                           &s->out2in.addr, s->out2in.port,
1439                                           s->nat_proto);
1440 }
1441
1442 nat44_ei_user_t *
1443 nat44_ei_user_get_or_create (nat44_ei_main_t *nm, ip4_address_t *addr,
1444                              u32 fib_index, u32 thread_index)
1445 {
1446   nat44_ei_user_t *u = 0;
1447   nat44_ei_user_key_t user_key;
1448   clib_bihash_kv_8_8_t kv, value;
1449   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1450   dlist_elt_t *per_user_list_head_elt;
1451
1452   user_key.addr.as_u32 = addr->as_u32;
1453   user_key.fib_index = fib_index;
1454   kv.key = user_key.as_u64;
1455
1456   /* Ever heard of the "user" = src ip4 address before? */
1457   if (clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1458     {
1459       if (pool_elts (tnm->users) >= nm->max_users_per_thread)
1460         {
1461           vlib_increment_simple_counter (&nm->user_limit_reached, thread_index,
1462                                          0, 1);
1463           nat_elog_warn (nm, "maximum user limit reached");
1464           return NULL;
1465         }
1466       /* no, make a new one */
1467       pool_get (tnm->users, u);
1468       clib_memset (u, 0, sizeof (*u));
1469
1470       u->addr.as_u32 = addr->as_u32;
1471       u->fib_index = fib_index;
1472
1473       pool_get (tnm->list_pool, per_user_list_head_elt);
1474
1475       u->sessions_per_user_list_head_index =
1476         per_user_list_head_elt - tnm->list_pool;
1477
1478       clib_dlist_init (tnm->list_pool, u->sessions_per_user_list_head_index);
1479
1480       kv.value = u - tnm->users;
1481
1482       /* add user */
1483       if (clib_bihash_add_del_8_8 (&tnm->user_hash, &kv, 1))
1484         {
1485           nat_elog_warn (nm, "user_hash key add failed");
1486           nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1487           return NULL;
1488         }
1489
1490       vlib_set_simple_counter (&nm->total_users, thread_index, 0,
1491                                pool_elts (tnm->users));
1492     }
1493   else
1494     {
1495       u = pool_elt_at_index (tnm->users, value.value);
1496     }
1497
1498   return u;
1499 }
1500
1501 nat44_ei_session_t *
1502 nat44_ei_session_alloc_or_recycle (nat44_ei_main_t *nm, nat44_ei_user_t *u,
1503                                    u32 thread_index, f64 now)
1504 {
1505   nat44_ei_session_t *s;
1506   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1507   u32 oldest_per_user_translation_list_index, session_index;
1508   dlist_elt_t *oldest_per_user_translation_list_elt;
1509   dlist_elt_t *per_user_translation_list_elt;
1510
1511   /* Over quota? Recycle the least recently used translation */
1512   if ((u->nsessions + u->nstaticsessions) >= nm->max_translations_per_user)
1513     {
1514       oldest_per_user_translation_list_index = clib_dlist_remove_head (
1515         tnm->list_pool, u->sessions_per_user_list_head_index);
1516
1517       ASSERT (oldest_per_user_translation_list_index != ~0);
1518
1519       /* Add it back to the end of the LRU list */
1520       clib_dlist_addtail (tnm->list_pool, u->sessions_per_user_list_head_index,
1521                           oldest_per_user_translation_list_index);
1522       /* Get the list element */
1523       oldest_per_user_translation_list_elt = pool_elt_at_index (
1524         tnm->list_pool, oldest_per_user_translation_list_index);
1525
1526       /* Get the session index from the list element */
1527       session_index = oldest_per_user_translation_list_elt->value;
1528
1529       /* Get the session */
1530       s = pool_elt_at_index (tnm->sessions, session_index);
1531
1532       nat44_ei_free_session_data_v2 (nm, s, thread_index, 0);
1533       if (nat44_ei_is_session_static (s))
1534         u->nstaticsessions--;
1535       else
1536         u->nsessions--;
1537       s->flags = 0;
1538       s->total_bytes = 0;
1539       s->total_pkts = 0;
1540       s->state = 0;
1541       s->ext_host_addr.as_u32 = 0;
1542       s->ext_host_port = 0;
1543       s->ext_host_nat_addr.as_u32 = 0;
1544       s->ext_host_nat_port = 0;
1545     }
1546   else
1547     {
1548       pool_get (tnm->sessions, s);
1549       clib_memset (s, 0, sizeof (*s));
1550
1551       /* Create list elts */
1552       pool_get (tnm->list_pool, per_user_translation_list_elt);
1553       clib_dlist_init (tnm->list_pool,
1554                        per_user_translation_list_elt - tnm->list_pool);
1555
1556       per_user_translation_list_elt->value = s - tnm->sessions;
1557       s->per_user_index = per_user_translation_list_elt - tnm->list_pool;
1558       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1559
1560       clib_dlist_addtail (tnm->list_pool, s->per_user_list_head_index,
1561                           per_user_translation_list_elt - tnm->list_pool);
1562
1563       s->user_index = u - tnm->users;
1564       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1565                                pool_elts (tnm->sessions));
1566     }
1567
1568   s->ha_last_refreshed = now;
1569
1570   return s;
1571 }
1572
1573 void
1574 nat44_ei_free_session_data (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1575                             u32 thread_index, u8 is_ha)
1576 {
1577   clib_bihash_kv_8_8_t kv;
1578
1579   init_nat_i2o_k (&kv, s);
1580   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1581     nat_elog_warn (nm, "in2out key del failed");
1582
1583   init_nat_o2i_k (&kv, s);
1584   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1585     nat_elog_warn (nm, "out2in key del failed");
1586
1587   if (!is_ha)
1588     {
1589       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1590                                &s->in2out.addr, s->in2out.port,
1591                                &s->out2in.addr, s->out2in.port, s->nat_proto);
1592
1593       nat_ipfix_logging_nat44_ses_delete (
1594         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1595         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1596         s->in2out.fib_index);
1597
1598       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1599                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1600                    thread_index);
1601     }
1602
1603   if (nat44_ei_is_session_static (s))
1604     return;
1605
1606   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1607                                           &s->out2in.addr, s->out2in.port,
1608                                           s->nat_proto);
1609 }
1610
1611 static_always_inline void
1612 nat44_ei_user_del_sessions (nat44_ei_user_t *u, u32 thread_index)
1613 {
1614   dlist_elt_t *elt;
1615   nat44_ei_session_t *s;
1616
1617   nat44_ei_main_t *nm = &nat44_ei_main;
1618   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1619
1620   // get head
1621   elt =
1622     pool_elt_at_index (tnm->list_pool, u->sessions_per_user_list_head_index);
1623   // get first element
1624   elt = pool_elt_at_index (tnm->list_pool, elt->next);
1625
1626   while (elt->value != ~0)
1627     {
1628       s = pool_elt_at_index (tnm->sessions, elt->value);
1629       elt = pool_elt_at_index (tnm->list_pool, elt->next);
1630
1631       nat44_ei_free_session_data (nm, s, thread_index, 0);
1632       nat44_ei_delete_session (nm, s, thread_index);
1633     }
1634 }
1635
1636 int
1637 nat44_ei_user_del (ip4_address_t *addr, u32 fib_index)
1638 {
1639   int rv = 1;
1640
1641   nat44_ei_main_t *nm = &nat44_ei_main;
1642   nat44_ei_main_per_thread_data_t *tnm;
1643
1644   nat44_ei_user_key_t user_key;
1645   clib_bihash_kv_8_8_t kv, value;
1646
1647   user_key.addr.as_u32 = addr->as_u32;
1648   user_key.fib_index = fib_index;
1649   kv.key = user_key.as_u64;
1650
1651   if (nm->num_workers > 1)
1652     {
1653       vec_foreach (tnm, nm->per_thread_data)
1654         {
1655           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1656             {
1657               nat44_ei_user_del_sessions (
1658                 pool_elt_at_index (tnm->users, value.value),
1659                 tnm->thread_index);
1660               rv = 0;
1661               break;
1662             }
1663         }
1664     }
1665   else
1666     {
1667       tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1668       if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1669         {
1670           nat44_ei_user_del_sessions (
1671             pool_elt_at_index (tnm->users, value.value), tnm->thread_index);
1672           rv = 0;
1673         }
1674     }
1675   return rv;
1676 }
1677
1678 void
1679 nat44_ei_static_mapping_del_sessions (nat44_ei_main_t *nm,
1680                                       nat44_ei_main_per_thread_data_t *tnm,
1681                                       nat44_ei_user_key_t u_key, int addr_only,
1682                                       ip4_address_t e_addr, u16 e_port)
1683 {
1684   clib_bihash_kv_8_8_t kv, value;
1685   kv.key = u_key.as_u64;
1686   u64 user_index;
1687   dlist_elt_t *head, *elt;
1688   nat44_ei_user_t *u;
1689   nat44_ei_session_t *s;
1690   u32 elt_index, head_index, ses_index;
1691
1692   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1693     {
1694       user_index = value.value;
1695       u = pool_elt_at_index (tnm->users, user_index);
1696       if (u->nstaticsessions)
1697         {
1698           head_index = u->sessions_per_user_list_head_index;
1699           head = pool_elt_at_index (tnm->list_pool, head_index);
1700           elt_index = head->next;
1701           elt = pool_elt_at_index (tnm->list_pool, elt_index);
1702           ses_index = elt->value;
1703           while (ses_index != ~0)
1704             {
1705               s = pool_elt_at_index (tnm->sessions, ses_index);
1706               elt = pool_elt_at_index (tnm->list_pool, elt->next);
1707               ses_index = elt->value;
1708
1709               if (!addr_only)
1710                 {
1711                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1712                       (s->out2in.port != e_port))
1713                     continue;
1714                 }
1715
1716               if (!nat44_ei_is_session_static (s))
1717                 continue;
1718
1719               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
1720                                              0);
1721               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1722
1723               if (!addr_only)
1724                 break;
1725             }
1726         }
1727     }
1728 }
1729
1730 u32
1731 nat44_ei_get_in2out_worker_index (ip4_header_t *ip0, u32 rx_fib_index0,
1732                                   u8 is_output)
1733 {
1734   nat44_ei_main_t *nm = &nat44_ei_main;
1735   u32 next_worker_index = 0;
1736   u32 hash;
1737
1738   next_worker_index = nm->first_worker_index;
1739   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
1740          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
1741
1742   if (PREDICT_TRUE (is_pow2 (_vec_len (nm->workers))))
1743     next_worker_index += nm->workers[hash & (_vec_len (nm->workers) - 1)];
1744   else
1745     next_worker_index += nm->workers[hash % _vec_len (nm->workers)];
1746
1747   return next_worker_index;
1748 }
1749
1750 u32
1751 nat44_ei_get_thread_idx_by_port (u16 e_port)
1752 {
1753   nat44_ei_main_t *nm = &nat44_ei_main;
1754   u32 thread_idx = nm->num_workers;
1755   if (nm->num_workers > 1)
1756     {
1757       thread_idx = nm->first_worker_index +
1758                    nm->workers[(e_port - 1024) / nm->port_per_thread %
1759                                _vec_len (nm->workers)];
1760     }
1761   return thread_idx;
1762 }
1763
1764 u32
1765 nat44_ei_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip0,
1766                                   u32 rx_fib_index0, u8 is_output)
1767 {
1768   nat44_ei_main_t *nm = &nat44_ei_main;
1769   udp_header_t *udp;
1770   u16 port;
1771   clib_bihash_kv_8_8_t kv, value;
1772   nat44_ei_static_mapping_t *m;
1773   u32 proto;
1774   u32 next_worker_index = 0;
1775
1776   /* first try static mappings without port */
1777   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1778     {
1779       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
1780       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1781                                    &value))
1782         {
1783           m = pool_elt_at_index (nm->static_mappings, value.value);
1784           return m->workers[0];
1785         }
1786     }
1787
1788   proto = ip_proto_to_nat_proto (ip0->protocol);
1789   udp = ip4_next_header (ip0);
1790   port = vnet_buffer (b)->ip.reass.l4_dst_port;
1791
1792   /* unknown protocol */
1793   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
1794     {
1795       /* use current thread */
1796       return vlib_get_thread_index ();
1797     }
1798
1799   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
1800     {
1801       icmp46_header_t *icmp = (icmp46_header_t *) udp;
1802       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
1803       if (!icmp_type_is_error_message (
1804             vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
1805         port = vnet_buffer (b)->ip.reass.l4_src_port;
1806       else
1807         {
1808           /* if error message, then it's not fragmented and we can access it */
1809           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
1810           proto = ip_proto_to_nat_proto (inner_ip->protocol);
1811           void *l4_header = ip4_next_header (inner_ip);
1812           switch (proto)
1813             {
1814             case NAT_PROTOCOL_ICMP:
1815               icmp = (icmp46_header_t *) l4_header;
1816               echo = (icmp_echo_header_t *) (icmp + 1);
1817               port = echo->identifier;
1818               break;
1819             case NAT_PROTOCOL_UDP:
1820             case NAT_PROTOCOL_TCP:
1821               port = ((tcp_udp_header_t *) l4_header)->src_port;
1822               break;
1823             default:
1824               return vlib_get_thread_index ();
1825             }
1826         }
1827     }
1828
1829   /* try static mappings with port */
1830   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1831     {
1832       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
1833       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1834                                    &value))
1835         {
1836           m = pool_elt_at_index (nm->static_mappings, value.value);
1837           return m->workers[0];
1838         }
1839     }
1840
1841   /* worker by outside port */
1842   next_worker_index =
1843     nat44_ei_get_thread_idx_by_port (clib_net_to_host_u16 (port));
1844   return next_worker_index;
1845 }
1846
1847 static int
1848 nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
1849                            u32 thread_index, nat_protocol_t proto,
1850                            ip4_address_t s_addr, ip4_address_t *addr,
1851                            u16 *port, u16 port_per_thread,
1852                            u32 snat_thread_index)
1853 {
1854   nat44_ei_main_t *nm = &nat44_ei_main;
1855   nat44_ei_address_t *a, *ga = 0;
1856   u32 portnum;
1857   int i;
1858
1859   if (vec_len (addresses) > 0)
1860     {
1861
1862       int s_addr_offset = s_addr.as_u32 % vec_len (addresses);
1863
1864       for (i = s_addr_offset; i < vec_len (addresses); ++i)
1865         {
1866           a = addresses + i;
1867           switch (proto)
1868             {
1869 #define _(N, j, n, s)                                                         \
1870   case NAT_PROTOCOL_##N:                                                      \
1871     if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread)       \
1872       {                                                                       \
1873         if (a->fib_index == fib_index)                                        \
1874           {                                                                   \
1875             while (1)                                                         \
1876               {                                                               \
1877                 portnum = (port_per_thread * snat_thread_index) +             \
1878                           nat_random_port (&nm->random_seed, 0,               \
1879                                            port_per_thread - 1) +             \
1880                           1024;                                               \
1881                 if (nat44_ei_port_is_used (a, NAT_PROTOCOL_##N, portnum))     \
1882                   continue;                                                   \
1883                 nat44_ei_port_get (a, NAT_PROTOCOL_##N, portnum);             \
1884                 a->busy_##n##_ports_per_thread[thread_index]++;               \
1885                 a->busy_##n##_ports++;                                        \
1886                 *addr = a->addr;                                              \
1887                 *port = clib_host_to_net_u16 (portnum);                       \
1888                 return 0;                                                     \
1889               }                                                               \
1890           }                                                                   \
1891         else if (a->fib_index == ~0)                                          \
1892           {                                                                   \
1893             ga = a;                                                           \
1894           }                                                                   \
1895       }                                                                       \
1896     break;
1897               foreach_nat_protocol;
1898             default:
1899               nat_elog_info (nm, "unknown protocol");
1900               return 1;
1901             }
1902         }
1903
1904       for (i = 0; i < s_addr_offset; ++i)
1905         {
1906           a = addresses + i;
1907           switch (proto)
1908             {
1909               foreach_nat_protocol;
1910             default:
1911               nat_elog_info (nm, "unknown protocol");
1912               return 1;
1913             }
1914         }
1915   if (ga)
1916     {
1917       a = ga;
1918       // fake fib index to reuse macro
1919       fib_index = ~0;
1920       switch (proto)
1921         {
1922           foreach_nat_protocol;
1923             default : nat_elog_info (nm, "unknown protocol");
1924           return 1;
1925         }
1926     }
1927     }
1928
1929 #undef _
1930
1931   /* Totally out of translations to use... */
1932   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1933   return 1;
1934 }
1935
1936 static int
1937 nat44_ei_alloc_range_cb (nat44_ei_address_t *addresses, u32 fib_index,
1938                          u32 thread_index, nat_protocol_t proto,
1939                          ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1940                          u16 port_per_thread, u32 snat_thread_index)
1941 {
1942   nat44_ei_main_t *nm = &nat44_ei_main;
1943   nat44_ei_address_t *a = addresses;
1944   u16 portnum, ports;
1945
1946   ports = nm->end_port - nm->start_port + 1;
1947
1948   if (!vec_len (addresses))
1949     goto exhausted;
1950
1951   switch (proto)
1952     {
1953 #define _(N, i, n, s)                                                         \
1954   case NAT_PROTOCOL_##N:                                                      \
1955     if (a->busy_##n##_ports < ports)                                          \
1956       {                                                                       \
1957         while (1)                                                             \
1958           {                                                                   \
1959             portnum = nat_random_port (&nm->random_seed, nm->start_port,      \
1960                                        nm->end_port);                         \
1961             if (nat44_ei_port_is_used (a, NAT_PROTOCOL_##N, portnum))         \
1962               continue;                                                       \
1963             nat44_ei_port_get (a, NAT_PROTOCOL_##N, portnum);                 \
1964             a->busy_##n##_ports++;                                            \
1965             *addr = a->addr;                                                  \
1966             *port = clib_host_to_net_u16 (portnum);                           \
1967             return 0;                                                         \
1968           }                                                                   \
1969       }                                                                       \
1970     break;
1971       foreach_nat_protocol
1972 #undef _
1973         default : nat_elog_info (nm, "unknown protocol");
1974       return 1;
1975     }
1976
1977 exhausted:
1978   /* Totally out of translations to use... */
1979   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1980   return 1;
1981 }
1982
1983 static int
1984 nat44_ei_alloc_mape_cb (nat44_ei_address_t *addresses, u32 fib_index,
1985                         u32 thread_index, nat_protocol_t proto,
1986                         ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1987                         u16 port_per_thread, u32 snat_thread_index)
1988 {
1989   nat44_ei_main_t *nm = &nat44_ei_main;
1990   nat44_ei_address_t *a = addresses;
1991   u16 m, ports, portnum, A, j;
1992   m = 16 - (nm->psid_offset + nm->psid_length);
1993   ports = (1 << (16 - nm->psid_length)) - (1 << m);
1994
1995   if (!vec_len (addresses))
1996     goto exhausted;
1997
1998   switch (proto)
1999     {
2000 #define _(N, i, n, s)                                                         \
2001   case NAT_PROTOCOL_##N:                                                      \
2002     if (a->busy_##n##_ports < ports)                                          \
2003       {                                                                       \
2004         while (1)                                                             \
2005           {                                                                   \
2006             A = nat_random_port (&nm->random_seed, 1,                         \
2007                                  pow2_mask (nm->psid_offset));                \
2008             j = nat_random_port (&nm->random_seed, 0, pow2_mask (m));         \
2009             portnum = A | (nm->psid << nm->psid_offset) | (j << (16 - m));    \
2010             if (nat44_ei_port_is_used (a, NAT_PROTOCOL_##N, portnum))         \
2011               continue;                                                       \
2012             nat44_ei_port_get (a, NAT_PROTOCOL_##N, portnum);                 \
2013             a->busy_##n##_ports++;                                            \
2014             *addr = a->addr;                                                  \
2015             *port = clib_host_to_net_u16 (portnum);                           \
2016             return 0;                                                         \
2017           }                                                                   \
2018       }                                                                       \
2019     break;
2020       foreach_nat_protocol
2021 #undef _
2022         default : nat_elog_info (nm, "unknown protocol");
2023       return 1;
2024     }
2025
2026 exhausted:
2027   /* Totally out of translations to use... */
2028   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
2029   return 1;
2030 }
2031
2032 void
2033 nat44_ei_set_alloc_default ()
2034 {
2035   nat44_ei_main_t *nm = &nat44_ei_main;
2036
2037   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2038   nm->alloc_addr_and_port = nat44_ei_alloc_default_cb;
2039 }
2040
2041 void
2042 nat44_ei_set_alloc_range (u16 start_port, u16 end_port)
2043 {
2044   nat44_ei_main_t *nm = &nat44_ei_main;
2045
2046   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_RANGE;
2047   nm->alloc_addr_and_port = nat44_ei_alloc_range_cb;
2048   nm->start_port = start_port;
2049   nm->end_port = end_port;
2050 }
2051
2052 void
2053 nat44_ei_set_alloc_mape (u16 psid, u16 psid_offset, u16 psid_length)
2054 {
2055   nat44_ei_main_t *nm = &nat44_ei_main;
2056
2057   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_MAPE;
2058   nm->alloc_addr_and_port = nat44_ei_alloc_mape_cb;
2059   nm->psid = psid;
2060   nm->psid_offset = psid_offset;
2061   nm->psid_length = psid_length;
2062 }
2063
2064 void
2065 nat44_ei_delete_session (nat44_ei_main_t *nm, nat44_ei_session_t *ses,
2066                          u32 thread_index)
2067 {
2068   nat44_ei_main_per_thread_data_t *tnm =
2069     vec_elt_at_index (nm->per_thread_data, thread_index);
2070   clib_bihash_kv_8_8_t kv, value;
2071   nat44_ei_user_t *u;
2072   const nat44_ei_user_key_t u_key = { .addr = ses->in2out.addr,
2073                                       .fib_index = ses->in2out.fib_index };
2074   const u8 u_static = nat44_ei_is_session_static (ses);
2075
2076   clib_dlist_remove (tnm->list_pool, ses->per_user_index);
2077   pool_put_index (tnm->list_pool, ses->per_user_index);
2078
2079   pool_put (tnm->sessions, ses);
2080   vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
2081                            pool_elts (tnm->sessions));
2082
2083   kv.key = u_key.as_u64;
2084   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2085     {
2086       u = pool_elt_at_index (tnm->users, value.value);
2087       if (u_static)
2088         u->nstaticsessions--;
2089       else
2090         u->nsessions--;
2091
2092       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
2093     }
2094 }
2095
2096 int
2097 nat44_ei_del_session (nat44_ei_main_t *nm, ip4_address_t *addr, u16 port,
2098                       nat_protocol_t proto, u32 vrf_id, int is_in)
2099 {
2100   nat44_ei_main_per_thread_data_t *tnm;
2101   clib_bihash_kv_8_8_t kv, value;
2102   u32 fib_index;
2103   nat44_ei_session_t *s;
2104   clib_bihash_8_8_t *t;
2105
2106   fail_if_disabled ();
2107
2108   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
2109   init_nat_k (&kv, *addr, port, fib_index, proto);
2110   t = is_in ? &nm->in2out : &nm->out2in;
2111   if (!clib_bihash_search_8_8 (t, &kv, &value))
2112     {
2113       // this is called from API/CLI, so the world is stopped here
2114       // it's safe to manipulate arbitrary per-thread data
2115       u32 thread_index = nat_value_get_thread_index (&value);
2116       tnm = vec_elt_at_index (nm->per_thread_data, thread_index);
2117       u32 session_index = nat_value_get_session_index (&value);
2118       if (pool_is_free_index (tnm->sessions, session_index))
2119         return VNET_API_ERROR_UNSPECIFIED;
2120
2121       s = pool_elt_at_index (tnm->sessions, session_index);
2122       nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data, 0);
2123       nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2124       return 0;
2125     }
2126
2127   return VNET_API_ERROR_NO_SUCH_ENTRY;
2128 }
2129
2130 void
2131 nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
2132                               int is_add)
2133 {
2134   nat44_ei_main_t *nm = &nat44_ei_main;
2135   fib_prefix_t prefix = {
2136     .fp_len = p_len,
2137     .fp_proto = FIB_PROTOCOL_IP4,
2138     .fp_addr = {
2139                 .ip4.as_u32 = addr->as_u32,
2140                 },
2141   };
2142   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2143
2144   if (is_add)
2145     {
2146       fib_table_entry_update_one_path (fib_index, &prefix, nm->fib_src_low,
2147                                        (FIB_ENTRY_FLAG_CONNECTED |
2148                                         FIB_ENTRY_FLAG_LOCAL |
2149                                         FIB_ENTRY_FLAG_EXCLUSIVE),
2150                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
2151                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
2152     }
2153   else
2154     {
2155       fib_table_entry_delete (fib_index, &prefix, nm->fib_src_low);
2156     }
2157 }
2158
2159 int
2160 nat44_ei_reserve_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2161 {
2162   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2163   nat44_ei_main_t *nm = &nat44_ei_main;
2164   nat44_ei_address_t *a = 0;
2165   int i;
2166
2167   for (i = 0; i < vec_len (nm->addresses); i++)
2168     {
2169       a = nm->addresses + i;
2170
2171       if (a->addr.as_u32 != addr.as_u32)
2172         continue;
2173
2174       switch (proto)
2175         {
2176 #define _(N, j, n, s)                                                         \
2177   case NAT_PROTOCOL_##N:                                                      \
2178     if (nat44_ei_port_is_used (a, NAT_PROTOCOL_##N, port))                    \
2179       continue;                                                               \
2180     nat44_ei_port_get (a, NAT_PROTOCOL_##N, port);                            \
2181     if (port > 1024)                                                          \
2182       {                                                                       \
2183         a->busy_##n##_ports++;                                                \
2184         a->busy_##n##_ports_per_thread[ti]++;                                 \
2185       }                                                                       \
2186     break;
2187           foreach_nat_protocol
2188 #undef _
2189             default : nat_elog_info (nm, "unknown protocol");
2190           goto done;
2191         }
2192
2193       return 0;
2194     }
2195
2196 done:
2197   return 1;
2198 }
2199
2200 int
2201 nat44_ei_free_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2202 {
2203   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2204   nat44_ei_main_t *nm = &nat44_ei_main;
2205   nat44_ei_address_t *a = 0;
2206   int i;
2207
2208   for (i = 0; i < vec_len (nm->addresses); i++)
2209     {
2210       a = nm->addresses + i;
2211
2212       if (a->addr.as_u32 != addr.as_u32)
2213         continue;
2214
2215       switch (proto)
2216         {
2217 #define _(N, j, n, s)                                                         \
2218   case NAT_PROTOCOL_##N:                                                      \
2219     nat44_ei_port_put (a, NAT_PROTOCOL_##N, port);                            \
2220     if (port > 1024)                                                          \
2221       {                                                                       \
2222         a->busy_##n##_ports--;                                                \
2223         a->busy_##n##_ports_per_thread[ti]--;                                 \
2224       }                                                                       \
2225     break;
2226           foreach_nat_protocol
2227 #undef _
2228             default : nat_elog_info (nm, "unknown protocol");
2229           goto done;
2230         }
2231
2232       return 0;
2233     }
2234
2235 done:
2236   return 1;
2237 }
2238
2239 void
2240 nat44_ei_add_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2241                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2242                              u32 flags, ip4_address_t pool_addr, u8 *tag)
2243 {
2244   nat44_ei_static_map_resolve_t *rp;
2245   nat44_ei_main_t *nm = &nat44_ei_main;
2246
2247   vec_add2 (nm->to_resolve, rp, 1);
2248   rp->l_addr.as_u32 = l_addr.as_u32;
2249   rp->l_port = l_port;
2250   rp->e_port = e_port;
2251   rp->sw_if_index = sw_if_index;
2252   rp->vrf_id = vrf_id;
2253   rp->proto = proto;
2254   rp->flags = flags;
2255   rp->pool_addr = pool_addr;
2256   rp->tag = vec_dup (tag);
2257 }
2258
2259 int
2260 nat44_ei_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2261                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2262                              u32 flags, int *out)
2263 {
2264   nat44_ei_static_map_resolve_t *rp;
2265   nat44_ei_main_t *nm = &nat44_ei_main;
2266   int i;
2267
2268   for (i = 0; i < vec_len (nm->to_resolve); i++)
2269     {
2270       rp = nm->to_resolve + i;
2271
2272       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
2273         {
2274           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
2275             {
2276               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2277                 {
2278                   if (rp->e_port != e_port || rp->proto != proto)
2279                     {
2280                       continue;
2281                     }
2282                 }
2283             }
2284           else if (rp->l_addr.as_u32 == l_addr.as_u32)
2285             {
2286               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2287                 {
2288                   if (rp->l_port != l_port || rp->e_port != e_port ||
2289                       rp->proto != proto)
2290                     {
2291                       continue;
2292                     }
2293                 }
2294             }
2295           else
2296             {
2297               continue;
2298             }
2299           if (out)
2300             {
2301               *out = i;
2302             }
2303           return 0;
2304         }
2305     }
2306   return 1;
2307 }
2308
2309 int
2310 nat44_ei_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2311                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2312                              u32 flags)
2313 {
2314   nat44_ei_main_t *nm = &nat44_ei_main;
2315   int i;
2316   if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2317                                     sw_if_index, flags, &i))
2318     {
2319       vec_del1 (nm->to_resolve, i);
2320       return 0;
2321     }
2322   return 1;
2323 }
2324
2325 void
2326 delete_matching_dynamic_sessions (const nat44_ei_static_mapping_t *m,
2327                                   u32 worker_index)
2328 {
2329   nat44_ei_main_t *nm = &nat44_ei_main;
2330   clib_bihash_kv_8_8_t kv, value;
2331   nat44_ei_session_t *s;
2332   nat44_ei_user_key_t u_key;
2333   nat44_ei_user_t *u;
2334   nat44_ei_main_per_thread_data_t *tnm;
2335   dlist_elt_t *head, *elt;
2336   u32 elt_index, head_index;
2337   u32 ses_index;
2338   u64 user_index;
2339
2340   if (nm->static_mapping_only)
2341     return;
2342
2343   tnm = vec_elt_at_index (nm->per_thread_data, worker_index);
2344
2345   u_key.addr = m->local_addr;
2346   u_key.fib_index = m->fib_index;
2347   kv.key = u_key.as_u64;
2348   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2349     {
2350       user_index = value.value;
2351       u = pool_elt_at_index (tnm->users, user_index);
2352       if (u->nsessions)
2353         {
2354           head_index = u->sessions_per_user_list_head_index;
2355           head = pool_elt_at_index (tnm->list_pool, head_index);
2356           elt_index = head->next;
2357           elt = pool_elt_at_index (tnm->list_pool, elt_index);
2358           ses_index = elt->value;
2359           while (ses_index != ~0)
2360             {
2361               s = pool_elt_at_index (tnm->sessions, ses_index);
2362               elt = pool_elt_at_index (tnm->list_pool, elt->next);
2363               ses_index = elt->value;
2364
2365               if (nat44_ei_is_session_static (s))
2366                 continue;
2367
2368               if (!is_sm_addr_only (m->flags) &&
2369                   s->in2out.port != m->local_port)
2370                 continue;
2371
2372               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
2373                                              0);
2374               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2375
2376               if (!is_sm_addr_only (m->flags))
2377                 break;
2378             }
2379         }
2380     }
2381 }
2382
2383 int
2384 nat44_ei_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2385                              u16 l_port, u16 e_port, nat_protocol_t proto,
2386                              u32 vrf_id, u32 sw_if_index, u32 flags,
2387                              ip4_address_t pool_addr, u8 *tag)
2388
2389 {
2390   nat44_ei_main_t *nm = &nat44_ei_main;
2391
2392   if (is_sm_switch_address (flags))
2393     {
2394       if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2395                                         sw_if_index, flags, 0))
2396         {
2397           return VNET_API_ERROR_VALUE_EXIST;
2398         }
2399
2400       nat44_ei_add_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2401                                    sw_if_index, flags, pool_addr, tag);
2402
2403       ip4_address_t *first_int_addr =
2404         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2405       if (!first_int_addr)
2406         {
2407           // dhcp resolution required
2408           return 0;
2409         }
2410
2411       e_addr.as_u32 = first_int_addr->as_u32;
2412     }
2413
2414   return nat44_ei_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
2415                                                proto, vrf_id, sw_if_index,
2416                                                flags, pool_addr, tag);
2417 }
2418
2419 int
2420 nat44_ei_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2421                              u16 l_port, u16 e_port, nat_protocol_t proto,
2422                              u32 vrf_id, u32 sw_if_index, u32 flags)
2423 {
2424   nat44_ei_main_t *nm = &nat44_ei_main;
2425
2426   if (is_sm_switch_address (flags))
2427     {
2428
2429       if (nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2430                                        sw_if_index, flags))
2431         {
2432           return VNET_API_ERROR_NO_SUCH_ENTRY;
2433         }
2434
2435       ip4_address_t *first_int_addr =
2436         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2437       if (!first_int_addr)
2438         {
2439           // dhcp resolution required
2440           return 0;
2441         }
2442
2443       e_addr.as_u32 = first_int_addr->as_u32;
2444     }
2445
2446   return nat44_ei_del_static_mapping_internal (
2447     l_addr, e_addr, l_port, e_port, proto, vrf_id, sw_if_index, flags);
2448 }
2449
2450 static int
2451 nat44_ei_add_static_mapping_internal (ip4_address_t l_addr,
2452                                       ip4_address_t e_addr, u16 l_port,
2453                                       u16 e_port, nat_protocol_t proto,
2454                                       u32 vrf_id, u32 sw_if_index, u32 flags,
2455                                       ip4_address_t pool_addr, u8 *tag)
2456 {
2457   nat44_ei_main_t *nm = &nat44_ei_main;
2458   clib_bihash_kv_8_8_t kv, value;
2459   nat44_ei_lb_addr_port_t *local;
2460   nat44_ei_static_mapping_t *m;
2461   u32 fib_index = ~0;
2462   u32 worker_index;
2463
2464   fail_if_disabled ();
2465
2466   if (is_sm_addr_only (flags))
2467     {
2468       e_port = l_port = proto = 0;
2469     }
2470
2471   if (is_sm_identity_nat (flags))
2472     {
2473       l_port = e_port;
2474       l_addr.as_u32 = e_addr.as_u32;
2475     }
2476
2477   // fib index 0
2478   init_nat_k (&kv, e_addr, e_port, 0, proto);
2479
2480   if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2481     {
2482       m = pool_elt_at_index (nm->static_mappings, value.value);
2483       if (!is_sm_identity_nat (m->flags))
2484         {
2485           return VNET_API_ERROR_VALUE_EXIST;
2486         }
2487
2488       // case:
2489       // adding local identity nat record for different vrf table
2490       pool_foreach (local, m->locals)
2491         {
2492           if (local->vrf_id == vrf_id)
2493             {
2494               return VNET_API_ERROR_VALUE_EXIST;
2495             }
2496         }
2497
2498       pool_get (m->locals, local);
2499
2500       local->vrf_id = vrf_id;
2501       local->fib_index = fib_table_find_or_create_and_lock (
2502         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2503
2504       init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
2505                    m->proto, 0, m - nm->static_mappings);
2506       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2507
2508       return 0;
2509     }
2510
2511   if (vrf_id != ~0)
2512     {
2513       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
2514                                                      nm->fib_src_low);
2515     }
2516   else
2517     {
2518       // fallback to default vrf
2519       vrf_id = nm->inside_vrf_id;
2520       fib_index = nm->inside_fib_index;
2521       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2522     }
2523
2524   if (!is_sm_identity_nat (flags))
2525     {
2526       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2527       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2528         {
2529           return VNET_API_ERROR_VALUE_EXIST;
2530         }
2531     }
2532
2533   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2534     {
2535       if (nat44_ei_reserve_port (e_addr, e_port, proto))
2536         {
2537           // remove resolve record
2538           if ((is_sm_switch_address (flags)) && !is_sm_identity_nat (flags))
2539             {
2540               nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto,
2541                                            vrf_id, sw_if_index, flags);
2542             }
2543           return VNET_API_ERROR_NO_SUCH_ENTRY;
2544         }
2545     }
2546
2547   pool_get (nm->static_mappings, m);
2548   clib_memset (m, 0, sizeof (*m));
2549
2550   m->flags = flags;
2551   m->local_addr = l_addr;
2552   m->external_addr = e_addr;
2553
2554   m->tag = vec_dup (tag);
2555
2556   if (!is_sm_addr_only (flags))
2557     {
2558       m->local_port = l_port;
2559       m->external_port = e_port;
2560       m->proto = proto;
2561     }
2562
2563   if (is_sm_identity_nat (flags))
2564     {
2565       pool_get (m->locals, local);
2566
2567       local->vrf_id = vrf_id;
2568       local->fib_index = fib_index;
2569     }
2570   else
2571     {
2572       m->vrf_id = vrf_id;
2573       m->fib_index = fib_index;
2574     }
2575
2576   init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
2577                m - nm->static_mappings);
2578   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2579
2580   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
2581                m - nm->static_mappings);
2582   clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 1);
2583
2584   if (nm->num_workers > 1)
2585     {
2586       // store worker index for this record
2587       ip4_header_t ip = {
2588         .src_address = m->local_addr,
2589       };
2590       worker_index = nat44_ei_get_in2out_worker_index (&ip, m->fib_index, 0);
2591       vec_add1 (m->workers, worker_index);
2592     }
2593   else
2594     {
2595       worker_index = nm->num_workers;
2596     }
2597   delete_matching_dynamic_sessions (m, worker_index);
2598
2599   if (is_sm_addr_only (flags))
2600     {
2601       nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 1);
2602     }
2603
2604   return 0;
2605 }
2606
2607 static int
2608 nat44_ei_del_static_mapping_internal (ip4_address_t l_addr,
2609                                       ip4_address_t e_addr, u16 l_port,
2610                                       u16 e_port, nat_protocol_t proto,
2611                                       u32 vrf_id, u32 sw_if_index, u32 flags)
2612 {
2613   nat44_ei_main_per_thread_data_t *tnm;
2614   nat44_ei_main_t *nm = &nat44_ei_main;
2615   clib_bihash_kv_8_8_t kv, value;
2616   nat44_ei_lb_addr_port_t *local;
2617   nat44_ei_static_mapping_t *m;
2618   u32 fib_index = ~0;
2619   nat44_ei_user_key_t u_key;
2620
2621   fail_if_disabled ();
2622
2623   if (is_sm_addr_only (flags))
2624     {
2625       e_port = l_port = proto = 0;
2626     }
2627
2628   if (is_sm_identity_nat (flags))
2629     {
2630       l_port = e_port;
2631       l_addr.as_u32 = e_addr.as_u32;
2632     }
2633
2634   // fib index 0
2635   init_nat_k (&kv, e_addr, e_port, 0, proto);
2636
2637   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2638     {
2639       if (is_sm_switch_address (flags))
2640         {
2641           return 0;
2642         }
2643       return VNET_API_ERROR_NO_SUCH_ENTRY;
2644     }
2645
2646   m = pool_elt_at_index (nm->static_mappings, value.value);
2647
2648   if (is_sm_identity_nat (flags))
2649     {
2650       u8 found = 0;
2651
2652       if (vrf_id == ~0)
2653         {
2654           vrf_id = nm->inside_vrf_id;
2655         }
2656
2657       pool_foreach (local, m->locals)
2658         {
2659           if (local->vrf_id == vrf_id)
2660             {
2661               local = pool_elt_at_index (m->locals, local - m->locals);
2662               fib_index = local->fib_index;
2663               pool_put (m->locals, local);
2664               found = 1;
2665             }
2666         }
2667       if (!found)
2668         {
2669           return VNET_API_ERROR_NO_SUCH_ENTRY;
2670         }
2671     }
2672   else
2673     {
2674       fib_index = m->fib_index;
2675     }
2676
2677   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2678     {
2679       if (nat44_ei_free_port (e_addr, e_port, proto))
2680         {
2681           return VNET_API_ERROR_INVALID_VALUE;
2682         }
2683     }
2684
2685   init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2686   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 0);
2687
2688   if (!nm->static_mapping_only || nm->static_mapping_connection_tracking)
2689     {
2690       // delete sessions for static mapping
2691       if (nm->num_workers > 1)
2692         tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2693       else
2694         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2695
2696       u_key.addr = m->local_addr;
2697       u_key.fib_index = fib_index;
2698       kv.key = u_key.as_u64;
2699       nat44_ei_static_mapping_del_sessions (
2700         nm, tnm, u_key, is_sm_addr_only (flags), e_addr, e_port);
2701     }
2702
2703   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2704
2705   if (!pool_elts (m->locals))
2706     {
2707       // this is last record remove all required stuff
2708       // fib_index 0
2709       init_nat_k (&kv, e_addr, e_port, 0, proto);
2710       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 0);
2711
2712       vec_free (m->tag);
2713       vec_free (m->workers);
2714       pool_put (nm->static_mappings, m);
2715
2716       if (is_sm_addr_only (flags) && !is_sm_identity_nat (flags))
2717         {
2718           nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 0);
2719         }
2720     }
2721
2722   return 0;
2723 }
2724
2725 int
2726 nat44_ei_static_mapping_match (ip4_address_t match_addr, u16 match_port,
2727                                u32 match_fib_index,
2728                                nat_protocol_t match_protocol,
2729                                ip4_address_t *mapping_addr, u16 *mapping_port,
2730                                u32 *mapping_fib_index, u8 by_external,
2731                                u8 *is_addr_only, u8 *is_identity_nat)
2732 {
2733   nat44_ei_main_t *nm = &nat44_ei_main;
2734   clib_bihash_kv_8_8_t kv, value;
2735   nat44_ei_static_mapping_t *m;
2736   u16 port;
2737
2738   if (by_external)
2739     {
2740       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2741       if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2742                                   &value))
2743         {
2744           /* Try address only mapping */
2745           init_nat_k (&kv, match_addr, 0, 0, 0);
2746           if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2747                                       &value))
2748             return 1;
2749         }
2750       m = pool_elt_at_index (nm->static_mappings, value.value);
2751
2752       *mapping_fib_index = m->fib_index;
2753       *mapping_addr = m->local_addr;
2754       port = m->local_port;
2755     }
2756   else
2757     {
2758       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2759                   match_protocol);
2760       if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2761         {
2762           /* Try address only mapping */
2763           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2764           if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
2765                                       &value))
2766             return 1;
2767         }
2768       m = pool_elt_at_index (nm->static_mappings, value.value);
2769
2770       *mapping_fib_index = nm->outside_fib_index;
2771       *mapping_addr = m->external_addr;
2772       port = m->external_port;
2773     }
2774
2775   /* Address only mapping doesn't change port */
2776   if (is_sm_addr_only (m->flags))
2777     *mapping_port = match_port;
2778   else
2779     *mapping_port = port;
2780
2781   if (PREDICT_FALSE (is_addr_only != 0))
2782     *is_addr_only = is_sm_addr_only (m->flags);
2783
2784   if (PREDICT_FALSE (is_identity_nat != 0))
2785     *is_identity_nat = is_sm_identity_nat (m->flags);
2786
2787   return 0;
2788 }
2789
2790 static void
2791 nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm)
2792 {
2793   pool_free (tnm->list_pool);
2794   pool_free (tnm->lru_pool);
2795   pool_free (tnm->sessions);
2796   pool_free (tnm->users);
2797
2798   clib_bihash_free_8_8 (&tnm->user_hash);
2799 }
2800
2801 u8 *
2802 format_nat44_ei_key (u8 *s, va_list *args)
2803 {
2804   u64 key = va_arg (*args, u64);
2805
2806   ip4_address_t addr;
2807   u16 port;
2808   nat_protocol_t protocol;
2809   u32 fib_index;
2810
2811   split_nat_key (key, &addr, &port, &fib_index, &protocol);
2812
2813   s = format (s, "%U proto %U port %d fib %d", format_ip4_address, &addr,
2814               format_nat_protocol, protocol, clib_net_to_host_u16 (port),
2815               fib_index);
2816   return s;
2817 }
2818
2819 u8 *
2820 format_nat44_ei_user_kvp (u8 *s, va_list *args)
2821 {
2822   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2823   nat44_ei_user_key_t k;
2824
2825   k.as_u64 = v->key;
2826
2827   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2828               k.fib_index, v->value);
2829
2830   return s;
2831 }
2832
2833 u8 *
2834 format_nat44_ei_session_kvp (u8 *s, va_list *args)
2835 {
2836   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2837
2838   s = format (s, "%U thread-index %llu session-index %llu",
2839               format_nat44_ei_key, v->key, nat_value_get_thread_index (v),
2840               nat_value_get_session_index (v));
2841
2842   return s;
2843 }
2844
2845 u8 *
2846 format_nat44_ei_static_mapping_kvp (u8 *s, va_list *args)
2847 {
2848   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2849
2850   s = format (s, "%U static-mapping-index %llu", format_nat44_ei_key, v->key,
2851               v->value);
2852
2853   return s;
2854 }
2855
2856 static void
2857 nat44_ei_worker_db_init (nat44_ei_main_per_thread_data_t *tnm,
2858                          u32 translations, u32 translation_buckets,
2859                          u32 user_buckets)
2860 {
2861   dlist_elt_t *head;
2862
2863   pool_alloc (tnm->list_pool, translations);
2864   pool_alloc (tnm->lru_pool, translations);
2865   pool_alloc (tnm->sessions, translations);
2866
2867   clib_bihash_init_8_8 (&tnm->user_hash, "users", user_buckets, 0);
2868
2869   clib_bihash_set_kvp_format_fn_8_8 (&tnm->user_hash,
2870                                      format_nat44_ei_user_kvp);
2871
2872   pool_get (tnm->lru_pool, head);
2873   tnm->tcp_trans_lru_head_index = head - tnm->lru_pool;
2874   clib_dlist_init (tnm->lru_pool, tnm->tcp_trans_lru_head_index);
2875
2876   pool_get (tnm->lru_pool, head);
2877   tnm->tcp_estab_lru_head_index = head - tnm->lru_pool;
2878   clib_dlist_init (tnm->lru_pool, tnm->tcp_estab_lru_head_index);
2879
2880   pool_get (tnm->lru_pool, head);
2881   tnm->udp_lru_head_index = head - tnm->lru_pool;
2882   clib_dlist_init (tnm->lru_pool, tnm->udp_lru_head_index);
2883
2884   pool_get (tnm->lru_pool, head);
2885   tnm->icmp_lru_head_index = head - tnm->lru_pool;
2886   clib_dlist_init (tnm->lru_pool, tnm->icmp_lru_head_index);
2887
2888   pool_get (tnm->lru_pool, head);
2889   tnm->unk_proto_lru_head_index = head - tnm->lru_pool;
2890   clib_dlist_init (tnm->lru_pool, tnm->unk_proto_lru_head_index);
2891 }
2892
2893 static void
2894 nat44_ei_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
2895 {
2896   nat44_ei_main_t *nm = &nat44_ei_main;
2897   nat44_ei_main_per_thread_data_t *tnm;
2898
2899   u32 static_mapping_buckets = 1024;
2900   u32 static_mapping_memory_size = 64 << 20;
2901
2902   clib_bihash_init_8_8 (&nm->static_mapping_by_local,
2903                         "static_mapping_by_local", static_mapping_buckets,
2904                         static_mapping_memory_size);
2905   clib_bihash_init_8_8 (&nm->static_mapping_by_external,
2906                         "static_mapping_by_external", static_mapping_buckets,
2907                         static_mapping_memory_size);
2908   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_local,
2909                                      format_nat44_ei_static_mapping_kvp);
2910   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_external,
2911                                      format_nat44_ei_static_mapping_kvp);
2912
2913   if (nm->pat)
2914     {
2915       clib_bihash_init_8_8 (&nm->in2out, "in2out", translation_buckets, 0);
2916       clib_bihash_init_8_8 (&nm->out2in, "out2in", translation_buckets, 0);
2917       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2918                                          format_nat44_ei_session_kvp);
2919       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2920                                          format_nat44_ei_session_kvp);
2921       vec_foreach (tnm, nm->per_thread_data)
2922         {
2923           nat44_ei_worker_db_init (tnm, translations, translation_buckets,
2924                                    user_buckets);
2925         }
2926     }
2927 }
2928
2929 void
2930 nat44_ei_sessions_clear ()
2931 {
2932   nat44_ei_main_t *nm = &nat44_ei_main;
2933   nat44_ei_main_per_thread_data_t *tnm;
2934
2935   if (nm->pat)
2936     {
2937       clib_bihash_free_8_8 (&nm->in2out);
2938       clib_bihash_free_8_8 (&nm->out2in);
2939       clib_bihash_init_8_8 (&nm->in2out, "in2out", nm->translation_buckets, 0);
2940       clib_bihash_init_8_8 (&nm->out2in, "out2in", nm->translation_buckets, 0);
2941       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2942                                          format_nat44_ei_session_kvp);
2943       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2944                                          format_nat44_ei_session_kvp);
2945       vec_foreach (tnm, nm->per_thread_data)
2946         {
2947           nat44_ei_worker_db_free (tnm);
2948           nat44_ei_worker_db_init (tnm, nm->translations,
2949                                    nm->translation_buckets, nm->user_buckets);
2950         }
2951     }
2952
2953   vlib_zero_simple_counter (&nm->total_users, 0);
2954   vlib_zero_simple_counter (&nm->total_sessions, 0);
2955   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
2956 }
2957
2958 static void
2959 nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque, u32 sw_if_index,
2960                              u32 new_fib_index, u32 old_fib_index)
2961 {
2962   nat44_ei_main_t *nm = &nat44_ei_main;
2963   nat44_ei_outside_fib_t *outside_fib;
2964   nat44_ei_interface_t *i;
2965   u8 is_add = 1;
2966   u8 match = 0;
2967
2968   if (!nm->enabled || (new_fib_index == old_fib_index) ||
2969       (!vec_len (nm->outside_fibs)))
2970     {
2971       return;
2972     }
2973
2974   pool_foreach (i, nm->interfaces)
2975     {
2976       if (i->sw_if_index == sw_if_index)
2977         {
2978           if (!(nat44_ei_interface_is_outside (i)))
2979             return;
2980           match = 1;
2981         }
2982     }
2983
2984   pool_foreach (i, nm->output_feature_interfaces)
2985     {
2986       if (i->sw_if_index == sw_if_index)
2987         {
2988           if (!(nat44_ei_interface_is_outside (i)))
2989             return;
2990           match = 1;
2991         }
2992     }
2993
2994   if (!match)
2995     return;
2996
2997   vec_foreach (outside_fib, nm->outside_fibs)
2998     {
2999       if (outside_fib->fib_index == old_fib_index)
3000         {
3001           outside_fib->refcount--;
3002           if (!outside_fib->refcount)
3003             vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
3004           break;
3005         }
3006     }
3007
3008   vec_foreach (outside_fib, nm->outside_fibs)
3009     {
3010       if (outside_fib->fib_index == new_fib_index)
3011         {
3012           outside_fib->refcount++;
3013           is_add = 0;
3014           break;
3015         }
3016     }
3017
3018   if (is_add)
3019     {
3020       vec_add2 (nm->outside_fibs, outside_fib, 1);
3021       outside_fib->refcount = 1;
3022       outside_fib->fib_index = new_fib_index;
3023     }
3024 }
3025
3026 int
3027 nat44_ei_add_address (ip4_address_t *addr, u32 vrf_id)
3028 {
3029   nat44_ei_main_t *nm = &nat44_ei_main;
3030   vlib_thread_main_t *tm = vlib_get_thread_main ();
3031   nat44_ei_address_t *ap;
3032
3033   fail_if_disabled ();
3034
3035   /* Check if address already exists */
3036   vec_foreach (ap, nm->addresses)
3037     {
3038       if (ap->addr.as_u32 == addr->as_u32)
3039         {
3040           nat44_ei_log_err ("address exist");
3041           return VNET_API_ERROR_VALUE_EXIST;
3042         }
3043     }
3044
3045   vec_add2 (nm->addresses, ap, 1);
3046
3047   ap->fib_index = ~0;
3048   ap->addr = *addr;
3049
3050   if (vrf_id != ~0)
3051     {
3052       ap->fib_index = fib_table_find_or_create_and_lock (
3053         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
3054     }
3055
3056 #define _(N, i, n, s)                                                         \
3057   ap->busy_##n##_port_bitmap = 0;                                             \
3058   ap->busy_##n##_ports = 0;                                                   \
3059   ap->busy_##n##_ports_per_thread = 0;                                        \
3060   vec_validate_init_empty (ap->busy_##n##_ports_per_thread,                   \
3061                            tm->n_vlib_mains - 1, 0);
3062   foreach_nat_protocol
3063 #undef _
3064
3065     nat44_ei_add_del_addr_to_fib_foreach_out_if (addr, 1);
3066
3067   return 0;
3068 }
3069
3070 int
3071 nat44_ei_del_address (ip4_address_t addr, u8 delete_sm)
3072 {
3073   nat44_ei_main_t *nm = &nat44_ei_main;
3074   nat44_ei_address_t *a = 0;
3075   nat44_ei_session_t *ses;
3076   u32 *ses_to_be_removed = 0, *ses_index;
3077   nat44_ei_main_per_thread_data_t *tnm;
3078   nat44_ei_static_mapping_t *m;
3079   int j;
3080
3081   fail_if_disabled ();
3082
3083   /* Find SNAT address */
3084   for (j = 0; j < vec_len (nm->addresses); j++)
3085     {
3086       if (nm->addresses[j].addr.as_u32 == addr.as_u32)
3087         {
3088           a = nm->addresses + j;
3089           break;
3090         }
3091     }
3092   if (!a)
3093     {
3094       nat44_ei_log_err ("no such address");
3095       return VNET_API_ERROR_NO_SUCH_ENTRY;
3096     }
3097
3098   if (delete_sm)
3099     {
3100       pool_foreach (m, nm->static_mappings)
3101         {
3102           if (m->external_addr.as_u32 == addr.as_u32)
3103             nat44_ei_del_static_mapping_internal (
3104               m->local_addr, m->external_addr, m->local_port, m->external_port,
3105               m->proto, m->vrf_id, ~0, m->flags);
3106         }
3107     }
3108   else
3109     {
3110       /* Check if address is used in some static mapping */
3111       if (nat44_ei_is_address_used_in_static_mapping (addr))
3112         {
3113           nat44_ei_log_err ("address used in static mapping");
3114           return VNET_API_ERROR_UNSPECIFIED;
3115         }
3116     }
3117
3118   /* Delete sessions using address */
3119   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
3120     {
3121       vec_foreach (tnm, nm->per_thread_data)
3122         {
3123           pool_foreach (ses, tnm->sessions)
3124             {
3125               if (ses->out2in.addr.as_u32 == addr.as_u32)
3126                 {
3127                   nat44_ei_free_session_data (nm, ses,
3128                                               tnm - nm->per_thread_data, 0);
3129                   vec_add1 (ses_to_be_removed, ses - tnm->sessions);
3130                 }
3131             }
3132           vec_foreach (ses_index, ses_to_be_removed)
3133             {
3134               ses = pool_elt_at_index (tnm->sessions, ses_index[0]);
3135               nat44_ei_delete_session (nm, ses, tnm - nm->per_thread_data);
3136             }
3137           vec_free (ses_to_be_removed);
3138         }
3139     }
3140
3141   nat44_ei_add_del_addr_to_fib_foreach_out_if (&addr, 0);
3142
3143   if (a->fib_index != ~0)
3144     {
3145       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
3146     }
3147
3148 #define _(N, i, n, s) vec_free (a->busy_##n##_ports_per_thread);
3149   foreach_nat_protocol
3150 #undef _
3151
3152     vec_del1 (nm->addresses, j);
3153   return 0;
3154 }
3155
3156 int
3157 nat44_ei_add_interface_address (u32 sw_if_index)
3158 {
3159   nat44_ei_main_t *nm = &nat44_ei_main;
3160   ip4_main_t *ip4_main = nm->ip4_main;
3161   ip4_address_t *first_int_addr;
3162   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3163   int i;
3164
3165   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3166     {
3167       if (auto_add_sw_if_indices[i] == sw_if_index)
3168         {
3169           return VNET_API_ERROR_VALUE_EXIST;
3170         }
3171     }
3172
3173   /* add to the auto-address list */
3174   vec_add1 (nm->auto_add_sw_if_indices, sw_if_index);
3175
3176   // if the address is already bound - or static - add it now
3177   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3178   if (first_int_addr)
3179     {
3180       (void) nat44_ei_add_address (first_int_addr, ~0);
3181     }
3182
3183   return 0;
3184 }
3185
3186 int
3187 nat44_ei_del_interface_address (u32 sw_if_index)
3188 {
3189   nat44_ei_main_t *nm = &nat44_ei_main;
3190   ip4_main_t *ip4_main = nm->ip4_main;
3191   ip4_address_t *first_int_addr;
3192   nat44_ei_static_map_resolve_t *rp;
3193   u32 *indices_to_delete = 0;
3194   int i, j;
3195   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3196
3197   fail_if_disabled ();
3198
3199   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3200
3201   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3202     {
3203       if (auto_add_sw_if_indices[i] == sw_if_index)
3204         {
3205           first_int_addr =
3206             ip4_interface_first_address (ip4_main, sw_if_index, 0);
3207           if (first_int_addr)
3208             {
3209               (void) nat44_ei_del_address (first_int_addr[0], 1);
3210             }
3211           else
3212             {
3213               for (j = 0; j < vec_len (nm->to_resolve); j++)
3214                 {
3215                   rp = nm->to_resolve + j;
3216                   if (rp->sw_if_index == sw_if_index)
3217                     {
3218                       vec_add1 (indices_to_delete, j);
3219                     }
3220                 }
3221               if (vec_len (indices_to_delete))
3222                 {
3223                   for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3224                     {
3225                       vec_del1 (nm->to_resolve, j);
3226                     }
3227                   vec_free (indices_to_delete);
3228                 }
3229             }
3230
3231           vec_del1 (nm->auto_add_sw_if_indices, i);
3232           return 0;
3233         }
3234     }
3235   return VNET_API_ERROR_NO_SUCH_ENTRY;
3236 }
3237
3238 static_always_inline int
3239 is_sw_if_index_reg_for_auto_resolve (u32 *sw_if_indices, u32 sw_if_index)
3240 {
3241   u32 *i;
3242   vec_foreach (i, sw_if_indices)
3243     {
3244       if (*i == sw_if_index)
3245         {
3246           return 1;
3247         }
3248     }
3249   return 0;
3250 }
3251
3252 static void
3253 nat44_ei_ip4_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3254                                            u32 sw_if_index,
3255                                            ip4_address_t *address,
3256                                            u32 address_length,
3257                                            u32 if_address_index, u32 is_delete)
3258 {
3259   nat44_ei_main_t *nm = &nat44_ei_main;
3260   nat44_ei_static_map_resolve_t *rp;
3261   nat44_ei_address_t *addresses = nm->addresses;
3262   int rv, i;
3263
3264   if (!nm->enabled)
3265     {
3266       return;
3267     }
3268
3269   if (!is_sw_if_index_reg_for_auto_resolve (nm->auto_add_sw_if_indices,
3270                                             sw_if_index))
3271     {
3272       return;
3273     }
3274
3275   if (!is_delete)
3276     {
3277       /* Don't trip over lease renewal, static config */
3278       for (i = 0; i < vec_len (addresses); i++)
3279         {
3280           if (addresses[i].addr.as_u32 == address->as_u32)
3281             {
3282               return;
3283             }
3284         }
3285
3286       (void) nat44_ei_add_address (address, ~0);
3287
3288       /* Scan static map resolution vector */
3289       for (i = 0; i < vec_len (nm->to_resolve); i++)
3290         {
3291           rp = nm->to_resolve + i;
3292           if (is_sm_addr_only (rp->flags))
3293             {
3294               continue;
3295             }
3296           /* On this interface? */
3297           if (rp->sw_if_index == sw_if_index)
3298             {
3299               rv = nat44_ei_add_static_mapping_internal (
3300                 rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3301                 rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3302               if (rv)
3303                 {
3304                   nat_elog_notice_X1 (
3305                     nm, "add_static_mapping_internal returned %d", "i4", rv);
3306                 }
3307             }
3308         }
3309     }
3310   else
3311     {
3312       // remove all static mapping records
3313       (void) nat44_ei_del_address (address[0], 1);
3314     }
3315 }
3316
3317 int
3318 nat44_ei_set_frame_queue_nelts (u32 frame_queue_nelts)
3319 {
3320   fail_if_enabled ();
3321   nat44_ei_main_t *nm = &nat44_ei_main;
3322   nm->frame_queue_nelts = frame_queue_nelts;
3323   return 0;
3324 }
3325
3326 static void
3327 nat44_ei_ip4_add_del_addr_only_sm_cb (ip4_main_t *im, uword opaque,
3328                                       u32 sw_if_index, ip4_address_t *address,
3329                                       u32 address_length, u32 if_address_index,
3330                                       u32 is_delete)
3331 {
3332   nat44_ei_main_t *nm = &nat44_ei_main;
3333   nat44_ei_static_map_resolve_t *rp;
3334   nat44_ei_static_mapping_t *m;
3335   clib_bihash_kv_8_8_t kv, value;
3336   int i, rv = 0, match = 0;
3337
3338   if (!nm->enabled)
3339     {
3340       return;
3341     }
3342
3343   for (i = 0; i < vec_len (nm->to_resolve); i++)
3344     {
3345       rp = nm->to_resolve + i;
3346
3347       if (is_sm_addr_only (rp->flags) && rp->sw_if_index == sw_if_index)
3348         {
3349           match = 1;
3350           break;
3351         }
3352     }
3353
3354   if (!match)
3355     {
3356       return;
3357     }
3358
3359   init_nat_k (&kv, *address, is_sm_addr_only (rp->flags) ? 0 : rp->e_port,
3360               nm->outside_fib_index,
3361               is_sm_addr_only (rp->flags) ? 0 : rp->proto);
3362   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
3363     m = 0;
3364   else
3365     m = pool_elt_at_index (nm->static_mappings, value.value);
3366
3367   if (is_delete)
3368     {
3369       if (!m)
3370         return;
3371       rv = nat44_ei_del_static_mapping_internal (
3372         rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto, rp->vrf_id,
3373         ~0, rp->flags);
3374       if (rv)
3375         {
3376           nat_elog_notice_X1 (nm, "nat44_ei_del_static_mapping returned %d",
3377                               "i4", rv);
3378         }
3379     }
3380   else
3381     {
3382       if (m)
3383         return;
3384       rv = nat44_ei_add_static_mapping_internal (
3385         rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto, rp->vrf_id,
3386         ~0, rp->flags, rp->pool_addr, rp->tag);
3387
3388       if (rv)
3389         {
3390           nat_elog_notice_X1 (nm, "nat44_ei_add_static_mapping returned %d",
3391                               "i4", rv);
3392         }
3393     }
3394 }
3395
3396 static_always_inline uword
3397 nat44_ei_classify_inline_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
3398                              vlib_frame_t *frame)
3399 {
3400   u32 n_left_from, *from, *to_next;
3401   nat44_ei_classify_next_t next_index;
3402   nat44_ei_main_t *nm = &nat44_ei_main;
3403   nat44_ei_static_mapping_t *m;
3404   u32 next_in2out = 0, next_out2in = 0;
3405
3406   from = vlib_frame_vector_args (frame);
3407   n_left_from = frame->n_vectors;
3408   next_index = node->cached_next_index;
3409
3410   while (n_left_from > 0)
3411     {
3412       u32 n_left_to_next;
3413
3414       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3415
3416       while (n_left_from > 0 && n_left_to_next > 0)
3417         {
3418           u32 bi0;
3419           vlib_buffer_t *b0;
3420           u32 next0 = NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3421           ip4_header_t *ip0;
3422           nat44_ei_address_t *ap;
3423           clib_bihash_kv_8_8_t kv0, value0;
3424
3425           /* speculatively enqueue b0 to the current next frame */
3426           bi0 = from[0];
3427           to_next[0] = bi0;
3428           from += 1;
3429           to_next += 1;
3430           n_left_from -= 1;
3431           n_left_to_next -= 1;
3432
3433           b0 = vlib_get_buffer (vm, bi0);
3434           ip0 = vlib_buffer_get_current (b0);
3435
3436           vec_foreach (ap, nm->addresses)
3437             {
3438               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
3439                 {
3440                   next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3441                   goto enqueue0;
3442                 }
3443             }
3444
3445           if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
3446             {
3447               init_nat_k (&kv0, ip0->dst_address, 0, 0, 0);
3448               /* try to classify the fragment based on IP header alone */
3449               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3450                                            &kv0, &value0))
3451                 {
3452                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3453                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3454                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3455                   goto enqueue0;
3456                 }
3457               init_nat_k (&kv0, ip0->dst_address,
3458                           vnet_buffer (b0)->ip.reass.l4_dst_port, 0,
3459                           ip_proto_to_nat_proto (ip0->protocol));
3460               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3461                                            &kv0, &value0))
3462                 {
3463                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3464                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3465                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3466                 }
3467             }
3468
3469         enqueue0:
3470           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
3471                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
3472             {
3473               nat44_ei_classify_trace_t *t =
3474                 vlib_add_trace (vm, node, b0, sizeof (*t));
3475               t->cached = 0;
3476               t->next_in2out = next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
3477             }
3478
3479           next_in2out += next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3480           next_out2in += next0 == NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3481
3482           /* verify speculative enqueue, maybe switch current next frame */
3483           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
3484                                            n_left_to_next, bi0, next0);
3485         }
3486
3487       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3488     }
3489
3490   vlib_node_increment_counter (
3491     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_IN2OUT, next_in2out);
3492   vlib_node_increment_counter (
3493     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_OUT2IN, next_out2in);
3494   return frame->n_vectors;
3495 }
3496
3497 VLIB_NODE_FN (nat44_ei_classify_node)
3498 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3499 {
3500   return nat44_ei_classify_inline_fn (vm, node, frame);
3501 }
3502
3503 VLIB_REGISTER_NODE (nat44_ei_classify_node) = {
3504   .name = "nat44-ei-classify",
3505   .vector_size = sizeof (u32),
3506   .format_trace = format_nat44_ei_classify_trace,
3507   .type = VLIB_NODE_TYPE_INTERNAL,
3508   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3509   .error_strings = nat44_ei_classify_error_strings,
3510   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3511   .next_nodes = {
3512     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out",
3513     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in",
3514     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3515   },
3516 };
3517
3518 VLIB_NODE_FN (nat44_ei_handoff_classify_node)
3519 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3520 {
3521   return nat44_ei_classify_inline_fn (vm, node, frame);
3522 }
3523
3524 VLIB_REGISTER_NODE (nat44_ei_handoff_classify_node) = {
3525   .name = "nat44-ei-handoff-classify",
3526   .vector_size = sizeof (u32),
3527   .format_trace = format_nat44_ei_classify_trace,
3528   .type = VLIB_NODE_TYPE_INTERNAL,
3529   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3530   .error_strings = nat44_ei_classify_error_strings,
3531   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3532   .next_nodes = {
3533     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out-worker-handoff",
3534     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in-worker-handoff",
3535     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3536   },
3537 };
3538
3539 /*
3540  * fd.io coding-style-patch-verification: ON
3541  *
3542  * Local Variables:
3543  * eval: (c-set-style "gnu")
3544  * End:
3545  */