src/plugins/nat/nat44-ei/nat44_ei.c

   1 /*
   2  * nat44_ei.c - nat44 endpoint dependent plugin
   3  *
   4  * Copyright (c) 2020 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  * License for the specific language governing permissions and limitations
  15  * under the License.
  16  */
  17
  18 #include <vnet/plugin/plugin.h>
  19 #include <vpp/app/version.h>
  20
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ip/ip4.h>
  24 #include <vnet/ip/ip_table.h>
  25 #include <vnet/ip/reass/ip4_sv_reass.h>
  26 #include <vnet/fib/fib_table.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <vnet/plugin/plugin.h>
  29
  30 // nat lib
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/nat_inlines.h>
  34 #include <nat/lib/ipfix_logging.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_dpo.h>
  37 #include <nat/nat44-ei/nat44_ei_inlines.h>
  38 #include <nat/nat44-ei/nat44_ei.h>
  39
  40 nat44_ei_main_t nat44_ei_main;
  41
  42 extern vlib_node_registration_t nat44_ei_hairpinning_node;
  43 extern vlib_node_registration_t
  44   nat44_ei_in2out_hairpinning_finish_ip4_lookup_node;
  45 extern vlib_node_registration_t
  46   nat44_ei_in2out_hairpinning_finish_interface_output_node;
  47
  48 #define skip_if_disabled()                                                    \
  49   do                                                                          \
  50     {                                                                         \
  51       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  52       if (PREDICT_FALSE (!nm->enabled))                                       \
  53         return;                                                               \
  54     }                                                                         \
  55   while (0)
  56
  57 #define fail_if_enabled()                                                     \
  58   do                                                                          \
  59     {                                                                         \
  60       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  61       if (PREDICT_FALSE (nm->enabled))                                        \
  62         {                                                                     \
  63           nat44_ei_log_err ("plugin enabled");                                \
  64           return VNET_API_ERROR_FEATURE_ALREADY_ENABLED;                      \
  65         }                                                                     \
  66     }                                                                         \
  67   while (0)
  68
  69 #define fail_if_disabled()                                                    \
  70   do                                                                          \
  71     {                                                                         \
  72       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  73       if (PREDICT_FALSE (!nm->enabled))                                       \
  74         {                                                                     \
  75           nat44_ei_log_err ("plugin disabled");                               \
  76           return VNET_API_ERROR_FEATURE_ALREADY_DISABLED;                     \
  77         }                                                                     \
  78     }                                                                         \
  79   while (0)
  80
  81 /* Hook up input features */
  82 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  83   .arc_name = "ip4-unicast",
  84   .node_name = "nat44-ei-classify",
  85   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  86                                "ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-ei-handoff-classify",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  92                                "ip4-sv-reassembly-feature"),
  93 };
  94 VNET_FEATURE_INIT (ip4_nat44_ei_in2out, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-ei-in2out",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  98                                "ip4-sv-reassembly-feature"),
  99 };
 100 VNET_FEATURE_INIT (ip4_nat44_ei_out2in, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-ei-out2in",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 104                                "ip4-sv-reassembly-feature",
 105                                "ip4-dhcp-client-detect"),
 106 };
 107 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output, static) = {
 108   .arc_name = "ip4-output",
 109   .node_name = "nat44-ei-in2out-output",
 110   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 111                                "ip4-sv-reassembly-output-feature"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ei_hairpinning, static) = {
 114   .arc_name = "ip4-local",
 115   .node_name = "nat44-ei-hairpinning",
 116   .runs_before = VNET_FEATURES ("ip4-local-end-of-arc"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_worker_handoff, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-ei-in2out-worker-handoff",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_worker_handoff, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ei-out2in-worker-handoff",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 127                                "ip4-dhcp-client-detect"),
 128 };
 129 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output_worker_handoff, static) = {
 130   .arc_name = "ip4-output",
 131   .node_name = "nat44-ei-in2out-output-worker-handoff",
 132   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 133                                "ip4-sv-reassembly-output-feature"),
 134 };
 135
 136 VLIB_PLUGIN_REGISTER () = {
 137   .version = VPP_BUILD_VER,
 138   .description = "IPv4 Endpoint-Independent NAT (NAT44 EI)",
 139 };
 140
 141 #define foreach_nat44_ei_classify_error                                       \
 142   _ (NEXT_IN2OUT, "next in2out")                                              \
 143   _ (NEXT_OUT2IN, "next out2in")                                              \
 144   _ (FRAG_CACHED, "fragment cached")
 145
 146 typedef enum
 147 {
 148 #define _(sym, str) NAT44_EI_CLASSIFY_ERROR_##sym,
 149   foreach_nat44_ei_classify_error
 150 #undef _
 151     NAT44_EI_CLASSIFY_N_ERROR,
 152 } nat44_ei_classify_error_t;
 153
 154 static char *nat44_ei_classify_error_strings[] = {
 155 #define _(sym, string) string,
 156   foreach_nat44_ei_classify_error
 157 #undef _
 158 };
 159
 160 typedef enum
 161 {
 162   NAT44_EI_CLASSIFY_NEXT_IN2OUT,
 163   NAT44_EI_CLASSIFY_NEXT_OUT2IN,
 164   NAT44_EI_CLASSIFY_NEXT_DROP,
 165   NAT44_EI_CLASSIFY_N_NEXT,
 166 } nat44_ei_classify_next_t;
 167
 168 typedef struct
 169 {
 170   u8 next_in2out;
 171   u8 cached;
 172 } nat44_ei_classify_trace_t;
 173
 174 void nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len,
 175                                    u32 sw_if_index, int is_add);
 176
 177 static void nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm);
 178
 179 static int nat44_ei_add_static_mapping_internal (
 180   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 181   nat_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 182   ip4_address_t pool_addr, u8 *tag);
 183
 184 static int nat44_ei_del_static_mapping_internal (
 185   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 186   nat_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags);
 187
 188 always_inline bool
 189 nat44_ei_port_is_used (nat44_ei_address_t *a, u8 proto, u16 port)
 190 {
 191   return clib_bitmap_get (a->busy_port_bitmap[proto], port);
 192 }
 193
 194 always_inline void
 195 nat44_ei_port_get (nat44_ei_address_t *a, u8 proto, u16 port)
 196 {
 197   ASSERT (!nat44_ei_port_is_used (a, proto, port));
 198   a->busy_port_bitmap[proto] =
 199     clib_bitmap_set (a->busy_port_bitmap[proto], port, 1);
 200 }
 201
 202 always_inline void
 203 nat44_ei_port_put (nat44_ei_address_t *a, u8 proto, u16 port)
 204 {
 205   ASSERT (nat44_ei_port_is_used (a, proto, port));
 206   a->busy_port_bitmap[proto] =
 207     clib_bitmap_set (a->busy_port_bitmap[proto], port, 0);
 208 }
 209
 210 static u8 *
 211 format_nat44_ei_classify_trace (u8 *s, va_list *args)
 212 {
 213   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 214   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 215   nat44_ei_classify_trace_t *t = va_arg (*args, nat44_ei_classify_trace_t *);
 216   char *next;
 217
 218   if (t->cached)
 219     s = format (s, "nat44-ei-classify: fragment cached");
 220   else
 221     {
 222       next = t->next_in2out ? "nat44-ei-in2out" : "nat44-ei-out2in";
 223       s = format (s, "nat44-ei-classify: next %s", next);
 224     }
 225
 226   return s;
 227 }
 228
 229 static void nat44_ei_db_init (u32 translations, u32 translation_buckets,
 230                               u32 user_buckets);
 231
 232 static void nat44_ei_ip4_add_del_interface_address_cb (
 233   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 234   u32 address_length, u32 if_address_index, u32 is_delete);
 235
 236 static void nat44_ei_ip4_add_del_addr_only_sm_cb (
 237   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 238   u32 address_length, u32 if_address_index, u32 is_delete);
 239
 240 static void nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque,
 241                                          u32 sw_if_index, u32 new_fib_index,
 242                                          u32 old_fib_index);
 243
 244 void
 245 nat44_ei_set_node_indexes (nat44_ei_main_t *nm, vlib_main_t *vm)
 246 {
 247   vlib_node_t *node;
 248   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-out2in");
 249   nm->out2in_node_index = node->index;
 250   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out");
 251   nm->in2out_node_index = node->index;
 252   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out-output");
 253   nm->in2out_output_node_index = node->index;
 254 }
 255
 256 int
 257 nat44_ei_set_workers (uword *bitmap)
 258 {
 259   nat44_ei_main_t *nm = &nat44_ei_main;
 260   int i, j = 0;
 261
 262   if (nm->num_workers < 2)
 263     return VNET_API_ERROR_FEATURE_DISABLED;
 264
 265   if (clib_bitmap_last_set (bitmap) >= nm->num_workers)
 266     return VNET_API_ERROR_INVALID_WORKER;
 267
 268   vec_free (nm->workers);
 269   clib_bitmap_foreach (i, bitmap)
 270     {
 271       vec_add1 (nm->workers, i);
 272       nm->per_thread_data[nm->first_worker_index + i].snat_thread_index = j;
 273       nm->per_thread_data[nm->first_worker_index + i].thread_index = i;
 274       j++;
 275     }
 276
 277   nm->port_per_thread = (0xffff - 1024) / _vec_len (nm->workers);
 278
 279   return 0;
 280 }
 281
 282 #define nat_validate_simple_counter(c, i)                                     \
 283   do                                                                          \
 284     {                                                                         \
 285       vlib_validate_simple_counter (&c, i);                                   \
 286       vlib_zero_simple_counter (&c, i);                                       \
 287     }                                                                         \
 288   while (0);
 289
 290 #define nat_init_simple_counter(c, n, sn)                                     \
 291   do                                                                          \
 292     {                                                                         \
 293       c.name = n;                                                             \
 294       c.stat_segment_name = sn;                                               \
 295       nat_validate_simple_counter (c, 0);                                     \
 296     }                                                                         \
 297   while (0);
 298
 299 static_always_inline void
 300 nat_validate_interface_counters (nat44_ei_main_t *nm, u32 sw_if_index)
 301 {
 302 #define _(x)                                                                  \
 303   nat_validate_simple_counter (nm->counters.fastpath.in2out.x, sw_if_index);  \
 304   nat_validate_simple_counter (nm->counters.fastpath.out2in.x, sw_if_index);  \
 305   nat_validate_simple_counter (nm->counters.slowpath.in2out.x, sw_if_index);  \
 306   nat_validate_simple_counter (nm->counters.slowpath.out2in.x, sw_if_index);
 307   foreach_nat_counter;
 308 #undef _
 309   nat_validate_simple_counter (nm->counters.hairpinning, sw_if_index);
 310 }
 311
 312 static void
 313 nat44_ei_add_del_addr_to_fib_foreach_out_if (ip4_address_t *addr, u8 is_add)
 314 {
 315   nat44_ei_main_t *nm = &nat44_ei_main;
 316   nat44_ei_interface_t *i;
 317
 318   pool_foreach (i, nm->interfaces)
 319     {
 320       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 321         {
 322           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 323         }
 324     }
 325   pool_foreach (i, nm->output_feature_interfaces)
 326     {
 327       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 328         {
 329           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 330         }
 331     }
 332 }
 333
 334 static_always_inline void
 335 nat44_ei_add_del_addr_to_fib_foreach_addr (u32 sw_if_index, u8 is_add)
 336 {
 337   nat44_ei_main_t *nm = &nat44_ei_main;
 338   nat44_ei_address_t *ap;
 339
 340   vec_foreach (ap, nm->addresses)
 341     {
 342       nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, is_add);
 343     }
 344 }
 345
 346 static_always_inline void
 347 nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (u32 sw_if_index, u8 is_add)
 348 {
 349   nat44_ei_main_t *nm = &nat44_ei_main;
 350   nat44_ei_static_mapping_t *m;
 351
 352   pool_foreach (m, nm->static_mappings)
 353     {
 354       if (is_sm_addr_only (m->flags) &&
 355           !(m->local_addr.as_u32 == m->external_addr.as_u32))
 356         {
 357           nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 358                                         is_add);
 359         }
 360     }
 361 }
 362
 363 static int
 364 nat44_ei_is_address_used_in_static_mapping (ip4_address_t addr)
 365 {
 366   nat44_ei_main_t *nm = &nat44_ei_main;
 367   nat44_ei_static_mapping_t *m;
 368   pool_foreach (m, nm->static_mappings)
 369     {
 370       if (is_sm_addr_only (m->flags) || is_sm_identity_nat (m->flags))
 371         {
 372           continue;
 373         }
 374       if (m->external_addr.as_u32 == addr.as_u32)
 375         {
 376           return 1;
 377         }
 378     }
 379   return 0;
 380 }
 381
 382 clib_error_t *
 383 nat44_ei_init (vlib_main_t *vm)
 384 {
 385   nat44_ei_main_t *nm = &nat44_ei_main;
 386   vlib_thread_main_t *tm = vlib_get_thread_main ();
 387   vlib_thread_registration_t *tr;
 388   ip4_add_del_interface_address_callback_t cbi = { 0 };
 389   ip4_table_bind_callback_t cbt = { 0 };
 390   u32 i, num_threads = 0;
 391   uword *p, *bitmap = 0;
 392
 393   clib_memset (nm, 0, sizeof (*nm));
 394
 395   // required
 396   nm->vnet_main = vnet_get_main ();
 397   // convenience
 398   nm->ip4_main = &ip4_main;
 399   nm->api_main = vlibapi_get_main ();
 400   nm->ip4_lookup_main = &ip4_main.lookup_main;
 401
 402   // handoff stuff
 403   nm->fq_out2in_index = ~0;
 404   nm->fq_in2out_index = ~0;
 405   nm->fq_in2out_output_index = ~0;
 406
 407   nm->log_level = NAT_LOG_ERROR;
 408
 409   nat44_ei_set_node_indexes (nm, vm);
 410   nm->log_class = vlib_log_register_class ("nat44-ei", 0);
 411
 412   nat_init_simple_counter (nm->total_users, "total-users",
 413                            "/nat44-ei/total-users");
 414   nat_init_simple_counter (nm->total_sessions, "total-sessions",
 415                            "/nat44-ei/total-sessions");
 416   nat_init_simple_counter (nm->user_limit_reached, "user-limit-reached",
 417                            "/nat44-ei/user-limit-reached");
 418
 419 #define _(x)                                                                  \
 420   nat_init_simple_counter (nm->counters.fastpath.in2out.x, #x,                \
 421                            "/nat44-ei/in2out/fastpath/" #x);                  \
 422   nat_init_simple_counter (nm->counters.fastpath.out2in.x, #x,                \
 423                            "/nat44-ei/out2in/fastpath/" #x);                  \
 424   nat_init_simple_counter (nm->counters.slowpath.in2out.x, #x,                \
 425                            "/nat44-ei/in2out/slowpath/" #x);                  \
 426   nat_init_simple_counter (nm->counters.slowpath.out2in.x, #x,                \
 427                            "/nat44-ei/out2in/slowpath/" #x);
 428   foreach_nat_counter;
 429 #undef _
 430   nat_init_simple_counter (nm->counters.hairpinning, "hairpinning",
 431                            "/nat44-ei/hairpinning");
 432
 433   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 434   if (p)
 435     {
 436       tr = (vlib_thread_registration_t *) p[0];
 437       if (tr)
 438         {
 439           nm->num_workers = tr->count;
 440           nm->first_worker_index = tr->first_index;
 441         }
 442     }
 443   num_threads = tm->n_vlib_mains - 1;
 444   nm->port_per_thread = 0xffff - 1024;
 445   vec_validate (nm->per_thread_data, num_threads);
 446
 447   /* Use all available workers by default */
 448   if (nm->num_workers > 1)
 449     {
 450       for (i = 0; i < nm->num_workers; i++)
 451         bitmap = clib_bitmap_set (bitmap, i, 1);
 452       nat44_ei_set_workers (bitmap);
 453       clib_bitmap_free (bitmap);
 454     }
 455   else
 456     {
 457       nm->per_thread_data[0].snat_thread_index = 0;
 458     }
 459
 460   /* callbacks to call when interface address changes. */
 461   cbi.function = nat44_ei_ip4_add_del_interface_address_cb;
 462   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 463   cbi.function = nat44_ei_ip4_add_del_addr_only_sm_cb;
 464   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 465
 466   /* callbacks to call when interface to table biding changes */
 467   cbt.function = nat44_ei_update_outside_fib;
 468   vec_add1 (nm->ip4_main->table_bind_callbacks, cbt);
 469
 470   nm->fib_src_low = fib_source_allocate (
 471     "nat44-ei-low", FIB_SOURCE_PRIORITY_LOW, FIB_SOURCE_BH_SIMPLE);
 472   nm->fib_src_hi = fib_source_allocate ("nat44-ei-hi", FIB_SOURCE_PRIORITY_HI,
 473                                         FIB_SOURCE_BH_SIMPLE);
 474
 475   // used only by out2in-dpo feature
 476   nat_dpo_module_init ();
 477   nat_ha_init (vm, nm->num_workers, num_threads);
 478
 479   nm->hairpinning_fq_index =
 480     vlib_frame_queue_main_init (nat44_ei_hairpinning_node.index, 0);
 481   nm->in2out_hairpinning_finish_ip4_lookup_node_fq_index =
 482     vlib_frame_queue_main_init (
 483       nat44_ei_in2out_hairpinning_finish_ip4_lookup_node.index, 0);
 484   nm->in2out_hairpinning_finish_interface_output_node_fq_index =
 485     vlib_frame_queue_main_init (
 486       nat44_ei_in2out_hairpinning_finish_interface_output_node.index, 0);
 487   return nat44_ei_api_hookup (vm);
 488 }
 489
 490 VLIB_INIT_FUNCTION (nat44_ei_init);
 491
 492 int
 493 nat44_ei_plugin_enable (nat44_ei_config_t c)
 494 {
 495   nat44_ei_main_t *nm = &nat44_ei_main;
 496
 497   fail_if_enabled ();
 498
 499   if (!c.users)
 500     c.users = 1024;
 501
 502   if (!c.sessions)
 503     c.sessions = 10 * 1024;
 504
 505   if (!c.user_sessions)
 506     c.user_sessions = c.sessions;
 507
 508   nm->rconfig = c;
 509
 510   if (!nm->frame_queue_nelts)
 511     nm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
 512
 513   nm->translations = c.sessions;
 514   nm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
 515   nm->user_buckets = nat_calc_bihash_buckets (c.users);
 516
 517   nm->pat = (!c.static_mapping_only ||
 518              (c.static_mapping_only && c.connection_tracking));
 519
 520   nm->static_mapping_only = c.static_mapping_only;
 521   nm->static_mapping_connection_tracking = c.connection_tracking;
 522   nm->out2in_dpo = c.out2in_dpo;
 523   nm->forwarding_enabled = 0;
 524   nm->mss_clamping = 0;
 525
 526   nm->max_users_per_thread = c.users;
 527   nm->max_translations_per_thread = c.sessions;
 528   nm->max_translations_per_user = c.user_sessions;
 529
 530   nm->inside_vrf_id = c.inside_vrf;
 531   nm->inside_fib_index = fib_table_find_or_create_and_lock (
 532     FIB_PROTOCOL_IP4, c.inside_vrf, nm->fib_src_hi);
 533
 534   nm->outside_vrf_id = c.outside_vrf;
 535   nm->outside_fib_index = fib_table_find_or_create_and_lock (
 536     FIB_PROTOCOL_IP4, c.outside_vrf, nm->fib_src_hi);
 537
 538   nat_reset_timeouts (&nm->timeouts);
 539   nat44_ei_db_init (nm->translations, nm->translation_buckets,
 540                     nm->user_buckets);
 541   nat44_ei_set_alloc_default ();
 542
 543   vlib_zero_simple_counter (&nm->total_users, 0);
 544   vlib_zero_simple_counter (&nm->total_sessions, 0);
 545   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
 546
 547   if (nm->num_workers > 1)
 548     {
 549       if (nm->fq_in2out_index == ~0)
 550         {
 551           nm->fq_in2out_index = vlib_frame_queue_main_init (
 552             nm->in2out_node_index, nm->frame_queue_nelts);
 553         }
 554       if (nm->fq_out2in_index == ~0)
 555         {
 556           nm->fq_out2in_index = vlib_frame_queue_main_init (
 557             nm->out2in_node_index, nm->frame_queue_nelts);
 558         }
 559       if (nm->fq_in2out_output_index == ~0)
 560         {
 561           nm->fq_in2out_output_index = vlib_frame_queue_main_init (
 562             nm->in2out_output_node_index, nm->frame_queue_nelts);
 563         }
 564     }
 565
 566   nat_ha_enable ();
 567   nm->enabled = 1;
 568
 569   return 0;
 570 }
 571
 572 static_always_inline nat44_ei_outside_fib_t *
 573 nat44_ei_get_outside_fib (nat44_ei_outside_fib_t *outside_fibs, u32 fib_index)
 574 {
 575   nat44_ei_outside_fib_t *f;
 576   vec_foreach (f, outside_fibs)
 577     {
 578       if (f->fib_index == fib_index)
 579         {
 580           return f;
 581         }
 582     }
 583   return 0;
 584 }
 585
 586 static_always_inline nat44_ei_interface_t *
 587 nat44_ei_get_interface (nat44_ei_interface_t *interfaces, u32 sw_if_index)
 588 {
 589   nat44_ei_interface_t *i;
 590   pool_foreach (i, interfaces)
 591     {
 592       if (i->sw_if_index == sw_if_index)
 593         {
 594           return i;
 595         }
 596     }
 597   return 0;
 598 }
 599
 600 static_always_inline int
 601 nat44_ei_hairpinning_enable (u8 is_enable)
 602 {
 603   nat44_ei_main_t *nm = &nat44_ei_main;
 604   u32 sw_if_index = 0; // local0
 605
 606   if (is_enable)
 607     {
 608       nm->hairpin_reg += 1;
 609       if (1 == nm->hairpin_reg)
 610         {
 611           return vnet_feature_enable_disable (
 612             "ip4-local", "nat44-ei-hairpinning", sw_if_index, is_enable, 0, 0);
 613         }
 614     }
 615   else
 616     {
 617       if (0 == nm->hairpin_reg)
 618         return 1;
 619
 620       nm->hairpin_reg -= 1;
 621       if (0 == nm->hairpin_reg)
 622         {
 623           return vnet_feature_enable_disable (
 624             "ip4-local", "nat44-ei-hairpinning", sw_if_index, is_enable, 0, 0);
 625         }
 626     }
 627
 628   return 0;
 629 }
 630
 631 int
 632 nat44_ei_add_interface (u32 sw_if_index, u8 is_inside)
 633 {
 634   const char *feature_name, *del_feature_name;
 635   nat44_ei_main_t *nm = &nat44_ei_main;
 636
 637   nat44_ei_outside_fib_t *outside_fib;
 638   nat44_ei_interface_t *i;
 639   u32 fib_index;
 640   int rv;
 641
 642   fail_if_disabled ();
 643
 644   if (nm->out2in_dpo && !is_inside)
 645     {
 646       nat44_ei_log_err ("error unsupported");
 647       return VNET_API_ERROR_UNSUPPORTED;
 648     }
 649
 650   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 651     {
 652       nat44_ei_log_err ("error interface already configured");
 653       return VNET_API_ERROR_VALUE_EXIST;
 654     }
 655
 656   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 657   if (i)
 658     {
 659       if ((nat44_ei_interface_is_inside (i) && is_inside) ||
 660           (nat44_ei_interface_is_outside (i) && !is_inside))
 661         {
 662           return 0;
 663         }
 664       if (nm->num_workers > 1)
 665         {
 666           del_feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 667                                           "nat44-ei-out2in-worker-handoff";
 668           feature_name = "nat44-ei-handoff-classify";
 669         }
 670       else
 671         {
 672           del_feature_name =
 673             !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 674
 675           feature_name = "nat44-ei-classify";
 676         }
 677
 678       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 679       if (rv)
 680         {
 681           return rv;
 682         }
 683       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 684                                         sw_if_index, 0, 0, 0);
 685       if (rv)
 686         {
 687           return rv;
 688         }
 689       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 690                                         sw_if_index, 1, 0, 0);
 691       if (rv)
 692         {
 693           return rv;
 694         }
 695       if (!is_inside)
 696         {
 697           rv = nat44_ei_hairpinning_enable (0);
 698           if (rv)
 699             {
 700               return rv;
 701             }
 702         }
 703     }
 704   else
 705     {
 706       if (nm->num_workers > 1)
 707         {
 708           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 709                                      "nat44-ei-out2in-worker-handoff";
 710         }
 711       else
 712         {
 713           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 714         }
 715       nat_validate_interface_counters (nm, sw_if_index);
 716
 717       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 718       if (rv)
 719         {
 720           return rv;
 721         }
 722       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 723                                         sw_if_index, 1, 0, 0);
 724       if (rv)
 725         {
 726           return rv;
 727         }
 728       if (is_inside && !nm->out2in_dpo)
 729         {
 730           rv = nat44_ei_hairpinning_enable (1);
 731           if (rv)
 732             {
 733               return rv;
 734             }
 735         }
 736
 737       pool_get (nm->interfaces, i);
 738       i->sw_if_index = sw_if_index;
 739       i->flags = 0;
 740     }
 741
 742   fib_index =
 743     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 744
 745   if (!is_inside)
 746     {
 747       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 748
 749       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 750       if (outside_fib)
 751         {
 752           outside_fib->refcount++;
 753         }
 754       else
 755         {
 756           vec_add2 (nm->outside_fibs, outside_fib, 1);
 757           outside_fib->fib_index = fib_index;
 758           outside_fib->refcount = 1;
 759         }
 760
 761       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 762       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 763     }
 764   else
 765     {
 766       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 767     }
 768
 769   return 0;
 770 }
 771
 772 int
 773 nat44_ei_del_interface (u32 sw_if_index, u8 is_inside)
 774 {
 775   const char *feature_name, *del_feature_name;
 776   nat44_ei_main_t *nm = &nat44_ei_main;
 777
 778   nat44_ei_outside_fib_t *outside_fib;
 779   nat44_ei_interface_t *i;
 780   u32 fib_index;
 781   int rv;
 782
 783   fail_if_disabled ();
 784
 785   if (nm->out2in_dpo && !is_inside)
 786     {
 787       nat44_ei_log_err ("error unsupported");
 788       return VNET_API_ERROR_UNSUPPORTED;
 789     }
 790
 791   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 792   if (i == 0)
 793     {
 794       nat44_ei_log_err ("error interface couldn't be found");
 795       return VNET_API_ERROR_NO_SUCH_ENTRY;
 796     }
 797
 798   if (nat44_ei_interface_is_inside (i) && nat44_ei_interface_is_outside (i))
 799     {
 800       if (nm->num_workers > 1)
 801         {
 802           del_feature_name = "nat44-ei-handoff-classify";
 803           feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 804                                       "nat44-ei-out2in-worker-handoff";
 805         }
 806       else
 807         {
 808           del_feature_name = "nat44-ei-classify";
 809           feature_name = !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 810         }
 811
 812       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 813       if (rv)
 814         {
 815           return rv;
 816         }
 817       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 818                                         sw_if_index, 0, 0, 0);
 819       if (rv)
 820         {
 821           return rv;
 822         }
 823       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 824                                         sw_if_index, 1, 0, 0);
 825       if (rv)
 826         {
 827           return rv;
 828         }
 829       if (is_inside)
 830         {
 831           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 832         }
 833       else
 834         {
 835           rv = nat44_ei_hairpinning_enable (1);
 836           if (rv)
 837             {
 838               return rv;
 839             }
 840           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 841         }
 842     }
 843   else
 844     {
 845       if (nm->num_workers > 1)
 846         {
 847           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 848                                      "nat44-ei-out2in-worker-handoff";
 849         }
 850       else
 851         {
 852           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 853         }
 854
 855       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 856       if (rv)
 857         {
 858           return rv;
 859         }
 860       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 861                                         sw_if_index, 0, 0, 0);
 862       if (rv)
 863         {
 864           return rv;
 865         }
 866       if (is_inside)
 867         {
 868           rv = nat44_ei_hairpinning_enable (0);
 869           if (rv)
 870             {
 871               return rv;
 872             }
 873         }
 874
 875       // remove interface
 876       pool_put (nm->interfaces, i);
 877     }
 878
 879   if (!is_inside)
 880     {
 881       fib_index =
 882         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 883       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 884       if (outside_fib)
 885         {
 886           outside_fib->refcount--;
 887           if (!outside_fib->refcount)
 888             {
 889               vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
 890             }
 891         }
 892
 893       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
 894       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
 895     }
 896
 897   return 0;
 898 }
 899
 900 int
 901 nat44_ei_add_output_interface (u32 sw_if_index)
 902 {
 903   nat44_ei_main_t *nm = &nat44_ei_main;
 904
 905   nat44_ei_outside_fib_t *outside_fib;
 906   nat44_ei_interface_t *i;
 907   u32 fib_index;
 908   int rv;
 909
 910   fail_if_disabled ();
 911
 912   if (nat44_ei_get_interface (nm->interfaces, sw_if_index))
 913     {
 914       nat44_ei_log_err ("error interface already configured");
 915       return VNET_API_ERROR_VALUE_EXIST;
 916     }
 917
 918   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 919     {
 920       nat44_ei_log_err ("error interface already configured");
 921       return VNET_API_ERROR_VALUE_EXIST;
 922     }
 923
 924   if (nm->num_workers > 1)
 925     {
 926       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 927       if (rv)
 928         {
 929           return rv;
 930         }
 931       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 932       if (rv)
 933         {
 934           return rv;
 935         }
 936       rv = vnet_feature_enable_disable (
 937         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 1, 0, 0);
 938       if (rv)
 939         {
 940           return rv;
 941         }
 942       rv = vnet_feature_enable_disable (
 943         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 1,
 944         0, 0);
 945       if (rv)
 946         {
 947           return rv;
 948         }
 949     }
 950   else
 951     {
 952       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 953       if (rv)
 954         {
 955           return rv;
 956         }
 957       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 958       if (rv)
 959         {
 960           return rv;
 961         }
 962       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
 963                                         sw_if_index, 1, 0, 0);
 964       if (rv)
 965         {
 966           return rv;
 967         }
 968       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
 969                                         sw_if_index, 1, 0, 0);
 970       if (rv)
 971         {
 972           return rv;
 973         }
 974     }
 975
 976   nat_validate_interface_counters (nm, sw_if_index);
 977
 978   pool_get (nm->output_feature_interfaces, i);
 979   i->sw_if_index = sw_if_index;
 980   i->flags = 0;
 981   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 982   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 983
 984   fib_index =
 985     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 986   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 987   if (outside_fib)
 988     {
 989       outside_fib->refcount++;
 990     }
 991   else
 992     {
 993       vec_add2 (nm->outside_fibs, outside_fib, 1);
 994       outside_fib->fib_index = fib_index;
 995       outside_fib->refcount = 1;
 996     }
 997
 998   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 999   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
1000
1001   return 0;
1002 }
1003
1004 int
1005 nat44_ei_del_output_interface (u32 sw_if_index)
1006 {
1007   nat44_ei_main_t *nm = &nat44_ei_main;
1008
1009   nat44_ei_outside_fib_t *outside_fib;
1010   nat44_ei_interface_t *i;
1011   u32 fib_index;
1012   int rv;
1013
1014   fail_if_disabled ();
1015
1016   i = nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index);
1017   if (!i)
1018     {
1019       nat44_ei_log_err ("error interface couldn't be found");
1020       return VNET_API_ERROR_NO_SUCH_ENTRY;
1021     }
1022
1023   if (nm->num_workers > 1)
1024     {
1025       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1026       if (rv)
1027         {
1028           return rv;
1029         }
1030       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1031       if (rv)
1032         {
1033           return rv;
1034         }
1035       rv = vnet_feature_enable_disable (
1036         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 0, 0, 0);
1037       if (rv)
1038         {
1039           return rv;
1040         }
1041       rv = vnet_feature_enable_disable (
1042         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 0,
1043         0, 0);
1044       if (rv)
1045         {
1046           return rv;
1047         }
1048     }
1049   else
1050     {
1051       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1052       if (rv)
1053         {
1054           return rv;
1055         }
1056       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1057       if (rv)
1058         {
1059           return rv;
1060         }
1061       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
1062                                         sw_if_index, 0, 0, 0);
1063       if (rv)
1064         {
1065           return rv;
1066         }
1067       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
1068                                         sw_if_index, 0, 0, 0);
1069       if (rv)
1070         {
1071           return rv;
1072         }
1073     }
1074
1075   pool_put (nm->output_feature_interfaces, i);
1076
1077   fib_index =
1078     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1079   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
1080   if (outside_fib)
1081     {
1082       outside_fib->refcount--;
1083       if (!outside_fib->refcount)
1084         {
1085           vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
1086         }
1087     }
1088
1089   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
1090   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
1091
1092   return 0;
1093 }
1094
1095 int
1096 nat44_ei_add_del_output_interface (u32 sw_if_index, int is_del)
1097 {
1098   if (is_del)
1099     {
1100       return nat44_ei_del_output_interface (sw_if_index);
1101     }
1102   else
1103     {
1104       return nat44_ei_add_output_interface (sw_if_index);
1105     }
1106 }
1107
1108 int
1109 nat44_ei_del_addresses ()
1110 {
1111   nat44_ei_main_t *nm = &nat44_ei_main;
1112   nat44_ei_address_t *a, *vec;
1113   int error = 0;
1114
1115   vec = vec_dup (nm->addresses);
1116   vec_foreach (a, vec)
1117     {
1118       error = nat44_ei_del_address (a->addr, 0);
1119
1120       if (error)
1121         {
1122           nat44_ei_log_err ("error occurred while removing adderess");
1123         }
1124     }
1125   vec_free (vec);
1126   vec_free (nm->addresses);
1127   nm->addresses = 0;
1128
1129   vec_free (nm->auto_add_sw_if_indices);
1130   nm->auto_add_sw_if_indices = 0;
1131   return error;
1132 }
1133
1134 int
1135 nat44_ei_del_interfaces ()
1136 {
1137   nat44_ei_main_t *nm = &nat44_ei_main;
1138   nat44_ei_interface_t *i, *pool;
1139   int error = 0;
1140
1141   pool = pool_dup (nm->interfaces);
1142   pool_foreach (i, pool)
1143     {
1144       if (nat44_ei_interface_is_inside (i))
1145         {
1146           error = nat44_ei_del_interface (i->sw_if_index, 1);
1147         }
1148       if (nat44_ei_interface_is_outside (i))
1149         {
1150           error = nat44_ei_del_interface (i->sw_if_index, 0);
1151         }
1152
1153       if (error)
1154         {
1155           nat44_ei_log_err ("error occurred while removing interface");
1156         }
1157     }
1158   pool_free (pool);
1159   pool_free (nm->interfaces);
1160   nm->interfaces = 0;
1161   return error;
1162 }
1163
1164 int
1165 nat44_ei_del_output_interfaces ()
1166 {
1167   nat44_ei_main_t *nm = &nat44_ei_main;
1168   nat44_ei_interface_t *i, *pool;
1169   int error = 0;
1170
1171   pool = pool_dup (nm->output_feature_interfaces);
1172   pool_foreach (i, pool)
1173     {
1174       error = nat44_ei_del_output_interface (i->sw_if_index);
1175       if (error)
1176         {
1177           nat44_ei_log_err ("error occurred while removing output interface");
1178         }
1179     }
1180   pool_free (pool);
1181   pool_free (nm->output_feature_interfaces);
1182   nm->output_feature_interfaces = 0;
1183   return error;
1184 }
1185
1186 int
1187 nat44_ei_del_static_mappings ()
1188 {
1189   nat44_ei_main_t *nm = &nat44_ei_main;
1190   nat44_ei_static_mapping_t *m, *pool;
1191   int error = 0;
1192
1193   pool = pool_dup (nm->static_mappings);
1194   pool_foreach (m, pool)
1195     {
1196       error = nat44_ei_del_static_mapping_internal (
1197         m->local_addr, m->external_addr, m->local_port, m->external_port,
1198         m->proto, m->vrf_id, ~0, m->flags);
1199       if (error)
1200         {
1201           nat44_ei_log_err ("error occurred while removing mapping");
1202         }
1203     }
1204   pool_free (pool);
1205   pool_free (nm->static_mappings);
1206   nm->static_mappings = 0;
1207
1208   vec_free (nm->to_resolve);
1209   nm->to_resolve = 0;
1210
1211   clib_bihash_free_8_8 (&nm->static_mapping_by_local);
1212   clib_bihash_free_8_8 (&nm->static_mapping_by_external);
1213
1214   return error;
1215 }
1216
1217 int
1218 nat44_ei_plugin_disable ()
1219 {
1220   nat44_ei_main_t *nm = &nat44_ei_main;
1221   nat44_ei_main_per_thread_data_t *tnm;
1222   int rc, error = 0;
1223
1224   fail_if_disabled ();
1225
1226   nat_ha_disable ();
1227
1228   rc = nat44_ei_del_static_mappings ();
1229   if (rc)
1230     error = VNET_API_ERROR_BUG;
1231
1232   rc = nat44_ei_del_addresses ();
1233   if (rc)
1234     error = VNET_API_ERROR_BUG;
1235
1236   rc = nat44_ei_del_interfaces ();
1237   if (rc)
1238     error = VNET_API_ERROR_BUG;
1239
1240   rc = nat44_ei_del_output_interfaces ();
1241   if (rc)
1242     error = VNET_API_ERROR_BUG;
1243
1244   if (nm->pat)
1245     {
1246       clib_bihash_free_8_8 (&nm->in2out);
1247       clib_bihash_free_8_8 (&nm->out2in);
1248
1249       vec_foreach (tnm, nm->per_thread_data)
1250         {
1251           nat44_ei_worker_db_free (tnm);
1252         }
1253     }
1254
1255   clib_memset (&nm->rconfig, 0, sizeof (nm->rconfig));
1256
1257   nm->forwarding_enabled = 0;
1258   nm->enabled = 0;
1259
1260   return error;
1261 }
1262
1263 int
1264 nat44_ei_set_outside_address_and_port (nat44_ei_address_t *addresses,
1265                                        u32 thread_index, ip4_address_t addr,
1266                                        u16 port, nat_protocol_t protocol)
1267 {
1268   nat44_ei_address_t *a = 0;
1269   u32 address_index;
1270   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1271
1272   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1273     {
1274       if (addresses[address_index].addr.as_u32 != addr.as_u32)
1275         continue;
1276
1277       a = addresses + address_index;
1278       if (nat44_ei_port_is_used (a, protocol, port_host_byte_order))
1279         return VNET_API_ERROR_INSTANCE_IN_USE;
1280
1281       nat44_ei_port_get (a, protocol, port_host_byte_order);
1282       a->busy_ports_per_thread[protocol][thread_index]++;
1283       a->busy_ports[protocol]++;
1284       return 0;
1285     }
1286
1287   return VNET_API_ERROR_NO_SUCH_ENTRY;
1288 }
1289
1290 void
1291 nat44_ei_add_del_address_dpo (ip4_address_t addr, u8 is_add)
1292 {
1293   nat44_ei_main_t *nm = &nat44_ei_main;
1294   dpo_id_t dpo_v4 = DPO_INVALID;
1295   fib_prefix_t pfx = {
1296     .fp_proto = FIB_PROTOCOL_IP4,
1297     .fp_len = 32,
1298     .fp_addr.ip4.as_u32 = addr.as_u32,
1299   };
1300
1301   if (is_add)
1302     {
1303       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
1304       fib_table_entry_special_dpo_add (0, &pfx, nm->fib_src_hi,
1305                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
1306       dpo_reset (&dpo_v4);
1307     }
1308   else
1309     {
1310       fib_table_entry_special_remove (0, &pfx, nm->fib_src_hi);
1311     }
1312 }
1313
1314 void
1315 nat44_ei_free_outside_address_and_port (nat44_ei_address_t *addresses,
1316                                         u32 thread_index, ip4_address_t *addr,
1317                                         u16 port, nat_protocol_t protocol)
1318 {
1319   nat44_ei_address_t *a;
1320   u32 address_index;
1321   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1322
1323   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1324     {
1325       if (addresses[address_index].addr.as_u32 == addr->as_u32)
1326         break;
1327     }
1328
1329   ASSERT (address_index < vec_len (addresses));
1330
1331   a = addresses + address_index;
1332   nat44_ei_port_put (a, protocol, port_host_byte_order);
1333   a->busy_ports[protocol]--;
1334   a->busy_ports_per_thread[protocol][thread_index]--;
1335 }
1336
1337 void
1338 nat44_ei_free_session_data_v2 (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1339                                u32 thread_index, u8 is_ha)
1340 {
1341   clib_bihash_kv_8_8_t kv;
1342
1343   /* session lookup tables */
1344   init_nat_i2o_k (&kv, s);
1345   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1346     nat_elog_warn (nm, "in2out key del failed");
1347   init_nat_o2i_k (&kv, s);
1348   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1349     nat_elog_warn (nm, "out2in key del failed");
1350
1351   if (!is_ha)
1352     nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1353                              &s->in2out.addr, s->in2out.port, &s->out2in.addr,
1354                              s->out2in.port, s->nat_proto);
1355
1356   if (nat44_ei_is_unk_proto_session (s))
1357     return;
1358
1359   if (!is_ha)
1360     {
1361       /* log NAT event */
1362       nat_ipfix_logging_nat44_ses_delete (
1363         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1364         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1365         s->in2out.fib_index);
1366
1367       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1368                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1369                    thread_index);
1370     }
1371
1372   if (nat44_ei_is_session_static (s))
1373     return;
1374
1375   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1376                                           &s->out2in.addr, s->out2in.port,
1377                                           s->nat_proto);
1378 }
1379
1380 nat44_ei_user_t *
1381 nat44_ei_user_get_or_create (nat44_ei_main_t *nm, ip4_address_t *addr,
1382                              u32 fib_index, u32 thread_index)
1383 {
1384   nat44_ei_user_t *u = 0;
1385   nat44_ei_user_key_t user_key;
1386   clib_bihash_kv_8_8_t kv, value;
1387   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1388   dlist_elt_t *per_user_list_head_elt;
1389
1390   user_key.addr.as_u32 = addr->as_u32;
1391   user_key.fib_index = fib_index;
1392   kv.key = user_key.as_u64;
1393
1394   /* Ever heard of the "user" = src ip4 address before? */
1395   if (clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1396     {
1397       if (pool_elts (tnm->users) >= nm->max_users_per_thread)
1398         {
1399           vlib_increment_simple_counter (&nm->user_limit_reached, thread_index,
1400                                          0, 1);
1401           nat_elog_warn (nm, "maximum user limit reached");
1402           return NULL;
1403         }
1404       /* no, make a new one */
1405       pool_get (tnm->users, u);
1406       clib_memset (u, 0, sizeof (*u));
1407
1408       u->addr.as_u32 = addr->as_u32;
1409       u->fib_index = fib_index;
1410
1411       pool_get (tnm->list_pool, per_user_list_head_elt);
1412
1413       u->sessions_per_user_list_head_index =
1414         per_user_list_head_elt - tnm->list_pool;
1415
1416       clib_dlist_init (tnm->list_pool, u->sessions_per_user_list_head_index);
1417
1418       kv.value = u - tnm->users;
1419
1420       /* add user */
1421       if (clib_bihash_add_del_8_8 (&tnm->user_hash, &kv, 1))
1422         {
1423           nat_elog_warn (nm, "user_hash key add failed");
1424           nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1425           return NULL;
1426         }
1427
1428       vlib_set_simple_counter (&nm->total_users, thread_index, 0,
1429                                pool_elts (tnm->users));
1430     }
1431   else
1432     {
1433       u = pool_elt_at_index (tnm->users, value.value);
1434     }
1435
1436   return u;
1437 }
1438
1439 nat44_ei_session_t *
1440 nat44_ei_session_alloc_or_recycle (nat44_ei_main_t *nm, nat44_ei_user_t *u,
1441                                    u32 thread_index, f64 now)
1442 {
1443   nat44_ei_session_t *s;
1444   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1445   u32 oldest_per_user_translation_list_index, session_index;
1446   dlist_elt_t *oldest_per_user_translation_list_elt;
1447   dlist_elt_t *per_user_translation_list_elt;
1448
1449   /* Over quota? Recycle the least recently used translation */
1450   if ((u->nsessions + u->nstaticsessions) >= nm->max_translations_per_user)
1451     {
1452       oldest_per_user_translation_list_index = clib_dlist_remove_head (
1453         tnm->list_pool, u->sessions_per_user_list_head_index);
1454
1455       ASSERT (oldest_per_user_translation_list_index != ~0);
1456
1457       /* Add it back to the end of the LRU list */
1458       clib_dlist_addtail (tnm->list_pool, u->sessions_per_user_list_head_index,
1459                           oldest_per_user_translation_list_index);
1460       /* Get the list element */
1461       oldest_per_user_translation_list_elt = pool_elt_at_index (
1462         tnm->list_pool, oldest_per_user_translation_list_index);
1463
1464       /* Get the session index from the list element */
1465       session_index = oldest_per_user_translation_list_elt->value;
1466
1467       /* Get the session */
1468       s = pool_elt_at_index (tnm->sessions, session_index);
1469
1470       nat44_ei_free_session_data_v2 (nm, s, thread_index, 0);
1471       if (nat44_ei_is_session_static (s))
1472         u->nstaticsessions--;
1473       else
1474         u->nsessions--;
1475       s->flags = 0;
1476       s->total_bytes = 0;
1477       s->total_pkts = 0;
1478       s->state = 0;
1479       s->ext_host_addr.as_u32 = 0;
1480       s->ext_host_port = 0;
1481       s->ext_host_nat_addr.as_u32 = 0;
1482       s->ext_host_nat_port = 0;
1483     }
1484   else
1485     {
1486       pool_get (tnm->sessions, s);
1487       clib_memset (s, 0, sizeof (*s));
1488
1489       /* Create list elts */
1490       pool_get (tnm->list_pool, per_user_translation_list_elt);
1491       clib_dlist_init (tnm->list_pool,
1492                        per_user_translation_list_elt - tnm->list_pool);
1493
1494       per_user_translation_list_elt->value = s - tnm->sessions;
1495       s->per_user_index = per_user_translation_list_elt - tnm->list_pool;
1496       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1497
1498       clib_dlist_addtail (tnm->list_pool, s->per_user_list_head_index,
1499                           per_user_translation_list_elt - tnm->list_pool);
1500
1501       s->user_index = u - tnm->users;
1502       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1503                                pool_elts (tnm->sessions));
1504     }
1505
1506   s->ha_last_refreshed = now;
1507
1508   return s;
1509 }
1510
1511 void
1512 nat44_ei_free_session_data (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1513                             u32 thread_index, u8 is_ha)
1514 {
1515   clib_bihash_kv_8_8_t kv;
1516
1517   init_nat_i2o_k (&kv, s);
1518   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1519     nat_elog_warn (nm, "in2out key del failed");
1520
1521   init_nat_o2i_k (&kv, s);
1522   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1523     nat_elog_warn (nm, "out2in key del failed");
1524
1525   if (!is_ha)
1526     {
1527       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1528                                &s->in2out.addr, s->in2out.port,
1529                                &s->out2in.addr, s->out2in.port, s->nat_proto);
1530
1531       nat_ipfix_logging_nat44_ses_delete (
1532         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1533         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1534         s->in2out.fib_index);
1535
1536       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1537                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1538                    thread_index);
1539     }
1540
1541   if (nat44_ei_is_session_static (s))
1542     return;
1543
1544   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1545                                           &s->out2in.addr, s->out2in.port,
1546                                           s->nat_proto);
1547 }
1548
1549 static_always_inline void
1550 nat44_ei_user_del_sessions (nat44_ei_user_t *u, u32 thread_index)
1551 {
1552   dlist_elt_t *elt;
1553   nat44_ei_session_t *s;
1554
1555   nat44_ei_main_t *nm = &nat44_ei_main;
1556   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1557
1558   // get head
1559   elt =
1560     pool_elt_at_index (tnm->list_pool, u->sessions_per_user_list_head_index);
1561   // get first element
1562   elt = pool_elt_at_index (tnm->list_pool, elt->next);
1563
1564   while (elt->value != ~0)
1565     {
1566       s = pool_elt_at_index (tnm->sessions, elt->value);
1567       elt = pool_elt_at_index (tnm->list_pool, elt->next);
1568
1569       nat44_ei_free_session_data (nm, s, thread_index, 0);
1570       nat44_ei_delete_session (nm, s, thread_index);
1571     }
1572 }
1573
1574 int
1575 nat44_ei_user_del (ip4_address_t *addr, u32 fib_index)
1576 {
1577   int rv = 1;
1578
1579   nat44_ei_main_t *nm = &nat44_ei_main;
1580   nat44_ei_main_per_thread_data_t *tnm;
1581
1582   nat44_ei_user_key_t user_key;
1583   clib_bihash_kv_8_8_t kv, value;
1584
1585   user_key.addr.as_u32 = addr->as_u32;
1586   user_key.fib_index = fib_index;
1587   kv.key = user_key.as_u64;
1588
1589   if (nm->num_workers > 1)
1590     {
1591       vec_foreach (tnm, nm->per_thread_data)
1592         {
1593           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1594             {
1595               nat44_ei_user_del_sessions (
1596                 pool_elt_at_index (tnm->users, value.value),
1597                 tnm->thread_index);
1598               rv = 0;
1599               break;
1600             }
1601         }
1602     }
1603   else
1604     {
1605       tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1606       if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1607         {
1608           nat44_ei_user_del_sessions (
1609             pool_elt_at_index (tnm->users, value.value), tnm->thread_index);
1610           rv = 0;
1611         }
1612     }
1613   return rv;
1614 }
1615
1616 void
1617 nat44_ei_static_mapping_del_sessions (nat44_ei_main_t *nm,
1618                                       nat44_ei_main_per_thread_data_t *tnm,
1619                                       nat44_ei_user_key_t u_key, int addr_only,
1620                                       ip4_address_t e_addr, u16 e_port)
1621 {
1622   clib_bihash_kv_8_8_t kv, value;
1623   kv.key = u_key.as_u64;
1624   u64 user_index;
1625   dlist_elt_t *head, *elt;
1626   nat44_ei_user_t *u;
1627   nat44_ei_session_t *s;
1628   u32 elt_index, head_index, ses_index;
1629
1630   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1631     {
1632       user_index = value.value;
1633       u = pool_elt_at_index (tnm->users, user_index);
1634       if (u->nstaticsessions)
1635         {
1636           head_index = u->sessions_per_user_list_head_index;
1637           head = pool_elt_at_index (tnm->list_pool, head_index);
1638           elt_index = head->next;
1639           elt = pool_elt_at_index (tnm->list_pool, elt_index);
1640           ses_index = elt->value;
1641           while (ses_index != ~0)
1642             {
1643               s = pool_elt_at_index (tnm->sessions, ses_index);
1644               elt = pool_elt_at_index (tnm->list_pool, elt->next);
1645               ses_index = elt->value;
1646
1647               if (!addr_only)
1648                 {
1649                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1650                       (s->out2in.port != e_port))
1651                     continue;
1652                 }
1653
1654               if (!nat44_ei_is_session_static (s))
1655                 continue;
1656
1657               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
1658                                              0);
1659               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1660
1661               if (!addr_only)
1662                 break;
1663             }
1664         }
1665     }
1666 }
1667
1668 u32
1669 nat44_ei_get_in2out_worker_index (ip4_header_t *ip0, u32 rx_fib_index0,
1670                                   u8 is_output)
1671 {
1672   nat44_ei_main_t *nm = &nat44_ei_main;
1673   u32 next_worker_index = 0;
1674   u32 hash;
1675
1676   next_worker_index = nm->first_worker_index;
1677   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
1678          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
1679
1680   if (PREDICT_TRUE (is_pow2 (_vec_len (nm->workers))))
1681     next_worker_index += nm->workers[hash & (_vec_len (nm->workers) - 1)];
1682   else
1683     next_worker_index += nm->workers[hash % _vec_len (nm->workers)];
1684
1685   return next_worker_index;
1686 }
1687
1688 u32
1689 nat44_ei_get_thread_idx_by_port (u16 e_port)
1690 {
1691   nat44_ei_main_t *nm = &nat44_ei_main;
1692   u32 thread_idx = nm->num_workers;
1693   if (nm->num_workers > 1)
1694     {
1695       thread_idx = nm->first_worker_index +
1696                    nm->workers[(e_port - 1024) / nm->port_per_thread %
1697                                _vec_len (nm->workers)];
1698     }
1699   return thread_idx;
1700 }
1701
1702 u32
1703 nat44_ei_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip0,
1704                                   u32 rx_fib_index0, u8 is_output)
1705 {
1706   nat44_ei_main_t *nm = &nat44_ei_main;
1707   udp_header_t *udp;
1708   u16 port;
1709   clib_bihash_kv_8_8_t kv, value;
1710   nat44_ei_static_mapping_t *m;
1711   u32 proto;
1712   u32 next_worker_index = 0;
1713
1714   /* first try static mappings without port */
1715   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1716     {
1717       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
1718       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1719                                    &value))
1720         {
1721           m = pool_elt_at_index (nm->static_mappings, value.value);
1722           return m->workers[0];
1723         }
1724     }
1725
1726   proto = ip_proto_to_nat_proto (ip0->protocol);
1727   udp = ip4_next_header (ip0);
1728   port = vnet_buffer (b)->ip.reass.l4_dst_port;
1729
1730   /* unknown protocol */
1731   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
1732     {
1733       /* use current thread */
1734       return vlib_get_thread_index ();
1735     }
1736
1737   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
1738     {
1739       icmp46_header_t *icmp = (icmp46_header_t *) udp;
1740       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
1741       if (!icmp_type_is_error_message (
1742             vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
1743         port = vnet_buffer (b)->ip.reass.l4_src_port;
1744       else
1745         {
1746           /* if error message, then it's not fragmented and we can access it */
1747           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
1748           proto = ip_proto_to_nat_proto (inner_ip->protocol);
1749           void *l4_header = ip4_next_header (inner_ip);
1750           switch (proto)
1751             {
1752             case NAT_PROTOCOL_ICMP:
1753               icmp = (icmp46_header_t *) l4_header;
1754               echo = (icmp_echo_header_t *) (icmp + 1);
1755               port = echo->identifier;
1756               break;
1757             case NAT_PROTOCOL_UDP:
1758             case NAT_PROTOCOL_TCP:
1759               port = ((tcp_udp_header_t *) l4_header)->src_port;
1760               break;
1761             default:
1762               return vlib_get_thread_index ();
1763             }
1764         }
1765     }
1766
1767   /* try static mappings with port */
1768   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1769     {
1770       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
1771       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1772                                    &value))
1773         {
1774           m = pool_elt_at_index (nm->static_mappings, value.value);
1775           return m->workers[0];
1776         }
1777     }
1778
1779   /* worker by outside port */
1780   next_worker_index =
1781     nat44_ei_get_thread_idx_by_port (clib_net_to_host_u16 (port));
1782   return next_worker_index;
1783 }
1784
1785 static int
1786 nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
1787                            u32 thread_index, nat_protocol_t proto,
1788                            ip4_address_t s_addr, ip4_address_t *addr,
1789                            u16 *port, u16 port_per_thread,
1790                            u32 snat_thread_index)
1791 {
1792   nat44_ei_main_t *nm = &nat44_ei_main;
1793   nat44_ei_address_t *a, *ga = 0;
1794   u32 portnum;
1795   int i;
1796
1797   if (vec_len (addresses) > 0)
1798     {
1799       int s_addr_offset = s_addr.as_u32 % vec_len (addresses);
1800
1801       for (i = s_addr_offset; i < vec_len (addresses); ++i)
1802         {
1803           a = addresses + i;
1804
1805           if (a->busy_ports_per_thread[proto][thread_index] < port_per_thread)
1806             {
1807               if (a->fib_index == fib_index)
1808                 {
1809                   while (1)
1810                     {
1811                       portnum = (port_per_thread * snat_thread_index) +
1812                                 nat_random_port (&nm->random_seed, 0,
1813                                                  port_per_thread - 1) +
1814                                 1024;
1815                       if (nat44_ei_port_is_used (a, proto, portnum))
1816                         continue;
1817                       nat44_ei_port_get (a, proto, portnum);
1818                       a->busy_ports_per_thread[proto][thread_index]++;
1819                       a->busy_ports[proto]++;
1820                       *addr = a->addr;
1821                       *port = clib_host_to_net_u16 (portnum);
1822                       return 0;
1823                     }
1824                 }
1825               else if (a->fib_index == ~0)
1826                 {
1827                   ga = a;
1828                 }
1829             }
1830         }
1831
1832       for (i = 0; i < s_addr_offset; ++i)
1833         {
1834           a = addresses + i;
1835           if (a->busy_ports_per_thread[proto][thread_index] < port_per_thread)
1836             {
1837               if (a->fib_index == fib_index)
1838                 {
1839                   while (1)
1840                     {
1841                       portnum = (port_per_thread * snat_thread_index) +
1842                                 nat_random_port (&nm->random_seed, 0,
1843                                                  port_per_thread - 1) +
1844                                 1024;
1845                       if (nat44_ei_port_is_used (a, proto, portnum))
1846                         continue;
1847                       nat44_ei_port_get (a, proto, portnum);
1848                       a->busy_ports_per_thread[proto][thread_index]++;
1849                       a->busy_ports[proto]++;
1850                       *addr = a->addr;
1851                       *port = clib_host_to_net_u16 (portnum);
1852                       return 0;
1853                     }
1854                 }
1855               else if (a->fib_index == ~0)
1856                 {
1857                   ga = a;
1858                 }
1859             }
1860         }
1861
1862       if (ga)
1863         {
1864           a = ga;
1865           if (a->busy_ports_per_thread[proto][thread_index] < port_per_thread)
1866             {
1867               if (a->fib_index == ~0)
1868                 {
1869                   while (1)
1870                     {
1871                       portnum = (port_per_thread * snat_thread_index) +
1872                                 nat_random_port (&nm->random_seed, 0,
1873                                                  port_per_thread - 1) +
1874                                 1024;
1875                       if (nat44_ei_port_is_used (a, proto, portnum))
1876                         continue;
1877                       nat44_ei_port_get (a, proto, portnum);
1878                       a->busy_ports_per_thread[proto][thread_index]++;
1879                       a->busy_ports[proto]++;
1880                       *addr = a->addr;
1881                       *port = clib_host_to_net_u16 (portnum);
1882                       return 0;
1883                     }
1884                 }
1885             }
1886         }
1887     }
1888
1889   /* Totally out of translations to use... */
1890   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1891   return 1;
1892 }
1893
1894 static int
1895 nat44_ei_alloc_range_cb (nat44_ei_address_t *addresses, u32 fib_index,
1896                          u32 thread_index, nat_protocol_t proto,
1897                          ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1898                          u16 port_per_thread, u32 snat_thread_index)
1899 {
1900   nat44_ei_main_t *nm = &nat44_ei_main;
1901   nat44_ei_address_t *a = addresses;
1902   u16 portnum, ports;
1903
1904   ports = nm->end_port - nm->start_port + 1;
1905
1906   if (!vec_len (addresses))
1907     goto exhausted;
1908
1909   if (a->busy_ports[proto] < ports)
1910     {
1911       while (1)
1912         {
1913           portnum =
1914             nat_random_port (&nm->random_seed, nm->start_port, nm->end_port);
1915           if (nat44_ei_port_is_used (a, proto, portnum))
1916             continue;
1917           nat44_ei_port_get (a, proto, portnum);
1918           a->busy_ports[proto]++;
1919           *addr = a->addr;
1920           *port = clib_host_to_net_u16 (portnum);
1921           return 0;
1922         }
1923     }
1924
1925 exhausted:
1926   /* Totally out of translations to use... */
1927   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1928   return 1;
1929 }
1930
1931 static int
1932 nat44_ei_alloc_mape_cb (nat44_ei_address_t *addresses, u32 fib_index,
1933                         u32 thread_index, nat_protocol_t proto,
1934                         ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1935                         u16 port_per_thread, u32 snat_thread_index)
1936 {
1937   nat44_ei_main_t *nm = &nat44_ei_main;
1938   nat44_ei_address_t *a = addresses;
1939   u16 m, ports, portnum, A, j;
1940   m = 16 - (nm->psid_offset + nm->psid_length);
1941   ports = (1 << (16 - nm->psid_length)) - (1 << m);
1942
1943   if (!vec_len (addresses))
1944     goto exhausted;
1945
1946   if (a->busy_ports[proto] < ports)
1947     {
1948       while (1)
1949         {
1950           A =
1951             nat_random_port (&nm->random_seed, 1, pow2_mask (nm->psid_offset));
1952           j = nat_random_port (&nm->random_seed, 0, pow2_mask (m));
1953           portnum = A | (nm->psid << nm->psid_offset) | (j << (16 - m));
1954           if (nat44_ei_port_is_used (a, proto, portnum))
1955             continue;
1956           nat44_ei_port_get (a, proto, portnum);
1957           a->busy_ports[proto]++;
1958           *addr = a->addr;
1959           *port = clib_host_to_net_u16 (portnum);
1960           return 0;
1961         }
1962     }
1963
1964 exhausted:
1965   /* Totally out of translations to use... */
1966   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1967   return 1;
1968 }
1969
1970 void
1971 nat44_ei_set_alloc_default ()
1972 {
1973   nat44_ei_main_t *nm = &nat44_ei_main;
1974
1975   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
1976   nm->alloc_addr_and_port = nat44_ei_alloc_default_cb;
1977 }
1978
1979 void
1980 nat44_ei_set_alloc_range (u16 start_port, u16 end_port)
1981 {
1982   nat44_ei_main_t *nm = &nat44_ei_main;
1983
1984   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_RANGE;
1985   nm->alloc_addr_and_port = nat44_ei_alloc_range_cb;
1986   nm->start_port = start_port;
1987   nm->end_port = end_port;
1988 }
1989
1990 void
1991 nat44_ei_set_alloc_mape (u16 psid, u16 psid_offset, u16 psid_length)
1992 {
1993   nat44_ei_main_t *nm = &nat44_ei_main;
1994
1995   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_MAPE;
1996   nm->alloc_addr_and_port = nat44_ei_alloc_mape_cb;
1997   nm->psid = psid;
1998   nm->psid_offset = psid_offset;
1999   nm->psid_length = psid_length;
2000 }
2001
2002 void
2003 nat44_ei_delete_session (nat44_ei_main_t *nm, nat44_ei_session_t *ses,
2004                          u32 thread_index)
2005 {
2006   nat44_ei_main_per_thread_data_t *tnm =
2007     vec_elt_at_index (nm->per_thread_data, thread_index);
2008   clib_bihash_kv_8_8_t kv, value;
2009   nat44_ei_user_t *u;
2010   const nat44_ei_user_key_t u_key = { .addr = ses->in2out.addr,
2011                                       .fib_index = ses->in2out.fib_index };
2012   const u8 u_static = nat44_ei_is_session_static (ses);
2013
2014   clib_dlist_remove (tnm->list_pool, ses->per_user_index);
2015   pool_put_index (tnm->list_pool, ses->per_user_index);
2016
2017   pool_put (tnm->sessions, ses);
2018   vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
2019                            pool_elts (tnm->sessions));
2020
2021   kv.key = u_key.as_u64;
2022   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2023     {
2024       u = pool_elt_at_index (tnm->users, value.value);
2025       if (u_static)
2026         u->nstaticsessions--;
2027       else
2028         u->nsessions--;
2029
2030       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
2031     }
2032 }
2033
2034 int
2035 nat44_ei_del_session (nat44_ei_main_t *nm, ip4_address_t *addr, u16 port,
2036                       nat_protocol_t proto, u32 vrf_id, int is_in)
2037 {
2038   nat44_ei_main_per_thread_data_t *tnm;
2039   clib_bihash_kv_8_8_t kv, value;
2040   u32 fib_index;
2041   nat44_ei_session_t *s;
2042   clib_bihash_8_8_t *t;
2043
2044   fail_if_disabled ();
2045
2046   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
2047   init_nat_k (&kv, *addr, port, fib_index, proto);
2048   t = is_in ? &nm->in2out : &nm->out2in;
2049   if (!clib_bihash_search_8_8 (t, &kv, &value))
2050     {
2051       // this is called from API/CLI, so the world is stopped here
2052       // it's safe to manipulate arbitrary per-thread data
2053       u32 thread_index = nat_value_get_thread_index (&value);
2054       tnm = vec_elt_at_index (nm->per_thread_data, thread_index);
2055       u32 session_index = nat_value_get_session_index (&value);
2056       if (pool_is_free_index (tnm->sessions, session_index))
2057         return VNET_API_ERROR_UNSPECIFIED;
2058
2059       s = pool_elt_at_index (tnm->sessions, session_index);
2060       nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data, 0);
2061       nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2062       return 0;
2063     }
2064
2065   return VNET_API_ERROR_NO_SUCH_ENTRY;
2066 }
2067
2068 void
2069 nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
2070                               int is_add)
2071 {
2072   nat44_ei_main_t *nm = &nat44_ei_main;
2073   fib_prefix_t prefix = {
2074     .fp_len = p_len,
2075     .fp_proto = FIB_PROTOCOL_IP4,
2076     .fp_addr = {
2077                 .ip4.as_u32 = addr->as_u32,
2078                 },
2079   };
2080   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2081
2082   if (is_add)
2083     {
2084       fib_table_entry_update_one_path (fib_index, &prefix, nm->fib_src_low,
2085                                        (FIB_ENTRY_FLAG_CONNECTED |
2086                                         FIB_ENTRY_FLAG_LOCAL |
2087                                         FIB_ENTRY_FLAG_EXCLUSIVE),
2088                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
2089                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
2090     }
2091   else
2092     {
2093       fib_table_entry_delete (fib_index, &prefix, nm->fib_src_low);
2094     }
2095 }
2096
2097 int
2098 nat44_ei_reserve_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2099 {
2100   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2101   nat44_ei_main_t *nm = &nat44_ei_main;
2102   nat44_ei_address_t *a = 0;
2103   int i;
2104
2105   for (i = 0; i < vec_len (nm->addresses); i++)
2106     {
2107       a = nm->addresses + i;
2108
2109       if (a->addr.as_u32 != addr.as_u32)
2110         continue;
2111
2112       if (nat44_ei_port_is_used (a, proto, port))
2113         continue;
2114
2115       nat44_ei_port_get (a, proto, port);
2116       if (port > 1024)
2117         {
2118           a->busy_ports[proto]++;
2119           a->busy_ports_per_thread[proto][ti]++;
2120         }
2121       return 0;
2122     }
2123
2124   return 1;
2125 }
2126
2127 int
2128 nat44_ei_free_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2129 {
2130   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2131   nat44_ei_main_t *nm = &nat44_ei_main;
2132   nat44_ei_address_t *a = 0;
2133   int i;
2134
2135   for (i = 0; i < vec_len (nm->addresses); i++)
2136     {
2137       a = nm->addresses + i;
2138
2139       if (a->addr.as_u32 != addr.as_u32)
2140         continue;
2141
2142       nat44_ei_port_put (a, proto, port);
2143       if (port > 1024)
2144         {
2145           a->busy_ports[proto]--;
2146           a->busy_ports_per_thread[proto][ti]--;
2147         }
2148       return 0;
2149     }
2150
2151   return 1;
2152 }
2153
2154 void
2155 nat44_ei_add_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2156                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2157                              u32 flags, ip4_address_t pool_addr, u8 *tag)
2158 {
2159   nat44_ei_static_map_resolve_t *rp;
2160   nat44_ei_main_t *nm = &nat44_ei_main;
2161
2162   vec_add2 (nm->to_resolve, rp, 1);
2163   rp->l_addr.as_u32 = l_addr.as_u32;
2164   rp->l_port = l_port;
2165   rp->e_port = e_port;
2166   rp->sw_if_index = sw_if_index;
2167   rp->vrf_id = vrf_id;
2168   rp->proto = proto;
2169   rp->flags = flags;
2170   rp->pool_addr = pool_addr;
2171   rp->tag = vec_dup (tag);
2172 }
2173
2174 int
2175 nat44_ei_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2176                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2177                              u32 flags, int *out)
2178 {
2179   nat44_ei_static_map_resolve_t *rp;
2180   nat44_ei_main_t *nm = &nat44_ei_main;
2181   int i;
2182
2183   for (i = 0; i < vec_len (nm->to_resolve); i++)
2184     {
2185       rp = nm->to_resolve + i;
2186
2187       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
2188         {
2189           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
2190             {
2191               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2192                 {
2193                   if (rp->e_port != e_port || rp->proto != proto)
2194                     {
2195                       continue;
2196                     }
2197                 }
2198             }
2199           else if (rp->l_addr.as_u32 == l_addr.as_u32)
2200             {
2201               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2202                 {
2203                   if (rp->l_port != l_port || rp->e_port != e_port ||
2204                       rp->proto != proto)
2205                     {
2206                       continue;
2207                     }
2208                 }
2209             }
2210           else
2211             {
2212               continue;
2213             }
2214           if (out)
2215             {
2216               *out = i;
2217             }
2218           return 0;
2219         }
2220     }
2221   return 1;
2222 }
2223
2224 int
2225 nat44_ei_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2226                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2227                              u32 flags)
2228 {
2229   nat44_ei_main_t *nm = &nat44_ei_main;
2230   int i;
2231   if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2232                                     sw_if_index, flags, &i))
2233     {
2234       vec_del1 (nm->to_resolve, i);
2235       return 0;
2236     }
2237   return 1;
2238 }
2239
2240 void
2241 delete_matching_dynamic_sessions (const nat44_ei_static_mapping_t *m,
2242                                   u32 worker_index)
2243 {
2244   nat44_ei_main_t *nm = &nat44_ei_main;
2245   clib_bihash_kv_8_8_t kv, value;
2246   nat44_ei_session_t *s;
2247   nat44_ei_user_key_t u_key;
2248   nat44_ei_user_t *u;
2249   nat44_ei_main_per_thread_data_t *tnm;
2250   dlist_elt_t *head, *elt;
2251   u32 elt_index, head_index;
2252   u32 ses_index;
2253   u64 user_index;
2254
2255   if (nm->static_mapping_only)
2256     return;
2257
2258   tnm = vec_elt_at_index (nm->per_thread_data, worker_index);
2259
2260   u_key.addr = m->local_addr;
2261   u_key.fib_index = m->fib_index;
2262   kv.key = u_key.as_u64;
2263   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2264     {
2265       user_index = value.value;
2266       u = pool_elt_at_index (tnm->users, user_index);
2267       if (u->nsessions)
2268         {
2269           head_index = u->sessions_per_user_list_head_index;
2270           head = pool_elt_at_index (tnm->list_pool, head_index);
2271           elt_index = head->next;
2272           elt = pool_elt_at_index (tnm->list_pool, elt_index);
2273           ses_index = elt->value;
2274           while (ses_index != ~0)
2275             {
2276               s = pool_elt_at_index (tnm->sessions, ses_index);
2277               elt = pool_elt_at_index (tnm->list_pool, elt->next);
2278               ses_index = elt->value;
2279
2280               if (nat44_ei_is_session_static (s))
2281                 continue;
2282
2283               if (!is_sm_addr_only (m->flags) &&
2284                   s->in2out.port != m->local_port)
2285                 continue;
2286
2287               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
2288                                              0);
2289               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2290
2291               if (!is_sm_addr_only (m->flags))
2292                 break;
2293             }
2294         }
2295     }
2296 }
2297
2298 int
2299 nat44_ei_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2300                              u16 l_port, u16 e_port, nat_protocol_t proto,
2301                              u32 vrf_id, u32 sw_if_index, u32 flags,
2302                              ip4_address_t pool_addr, u8 *tag)
2303
2304 {
2305   nat44_ei_main_t *nm = &nat44_ei_main;
2306
2307   if (is_sm_switch_address (flags))
2308     {
2309       if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2310                                         sw_if_index, flags, 0))
2311         {
2312           return VNET_API_ERROR_VALUE_EXIST;
2313         }
2314
2315       nat44_ei_add_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2316                                    sw_if_index, flags, pool_addr, tag);
2317
2318       ip4_address_t *first_int_addr =
2319         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2320       if (!first_int_addr)
2321         {
2322           // dhcp resolution required
2323           return 0;
2324         }
2325
2326       e_addr.as_u32 = first_int_addr->as_u32;
2327     }
2328
2329   return nat44_ei_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
2330                                                proto, vrf_id, sw_if_index,
2331                                                flags, pool_addr, tag);
2332 }
2333
2334 int
2335 nat44_ei_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2336                              u16 l_port, u16 e_port, nat_protocol_t proto,
2337                              u32 vrf_id, u32 sw_if_index, u32 flags)
2338 {
2339   nat44_ei_main_t *nm = &nat44_ei_main;
2340
2341   if (is_sm_switch_address (flags))
2342     {
2343
2344       if (nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2345                                        sw_if_index, flags))
2346         {
2347           return VNET_API_ERROR_NO_SUCH_ENTRY;
2348         }
2349
2350       ip4_address_t *first_int_addr =
2351         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2352       if (!first_int_addr)
2353         {
2354           // dhcp resolution required
2355           return 0;
2356         }
2357
2358       e_addr.as_u32 = first_int_addr->as_u32;
2359     }
2360
2361   return nat44_ei_del_static_mapping_internal (
2362     l_addr, e_addr, l_port, e_port, proto, vrf_id, sw_if_index, flags);
2363 }
2364
2365 static int
2366 nat44_ei_add_static_mapping_internal (ip4_address_t l_addr,
2367                                       ip4_address_t e_addr, u16 l_port,
2368                                       u16 e_port, nat_protocol_t proto,
2369                                       u32 vrf_id, u32 sw_if_index, u32 flags,
2370                                       ip4_address_t pool_addr, u8 *tag)
2371 {
2372   nat44_ei_main_t *nm = &nat44_ei_main;
2373   clib_bihash_kv_8_8_t kv, value;
2374   nat44_ei_lb_addr_port_t *local;
2375   nat44_ei_static_mapping_t *m;
2376   u32 fib_index = ~0;
2377   u32 worker_index;
2378
2379   fail_if_disabled ();
2380
2381   if (is_sm_addr_only (flags))
2382     {
2383       e_port = l_port = proto = 0;
2384     }
2385
2386   if (is_sm_identity_nat (flags))
2387     {
2388       l_port = e_port;
2389       l_addr.as_u32 = e_addr.as_u32;
2390     }
2391
2392   // fib index 0
2393   init_nat_k (&kv, e_addr, e_port, 0, proto);
2394
2395   if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2396     {
2397       m = pool_elt_at_index (nm->static_mappings, value.value);
2398       if (!is_sm_identity_nat (m->flags))
2399         {
2400           return VNET_API_ERROR_VALUE_EXIST;
2401         }
2402
2403       // case:
2404       // adding local identity nat record for different vrf table
2405       pool_foreach (local, m->locals)
2406         {
2407           if (local->vrf_id == vrf_id)
2408             {
2409               return VNET_API_ERROR_VALUE_EXIST;
2410             }
2411         }
2412
2413       pool_get (m->locals, local);
2414
2415       local->vrf_id = vrf_id;
2416       local->fib_index = fib_table_find_or_create_and_lock (
2417         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2418
2419       init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
2420                    m->proto, 0, m - nm->static_mappings);
2421       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2422
2423       return 0;
2424     }
2425
2426   if (vrf_id != ~0)
2427     {
2428       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
2429                                                      nm->fib_src_low);
2430     }
2431   else
2432     {
2433       // fallback to default vrf
2434       vrf_id = nm->inside_vrf_id;
2435       fib_index = nm->inside_fib_index;
2436       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2437     }
2438
2439   if (!is_sm_identity_nat (flags))
2440     {
2441       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2442       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2443         {
2444           return VNET_API_ERROR_VALUE_EXIST;
2445         }
2446     }
2447
2448   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2449     {
2450       if (nat44_ei_reserve_port (e_addr, e_port, proto))
2451         {
2452           // remove resolve record
2453           if ((is_sm_switch_address (flags)) && !is_sm_identity_nat (flags))
2454             {
2455               nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto,
2456                                            vrf_id, sw_if_index, flags);
2457             }
2458           return VNET_API_ERROR_NO_SUCH_ENTRY;
2459         }
2460     }
2461
2462   pool_get (nm->static_mappings, m);
2463   clib_memset (m, 0, sizeof (*m));
2464
2465   m->flags = flags;
2466   m->local_addr = l_addr;
2467   m->external_addr = e_addr;
2468
2469   m->tag = vec_dup (tag);
2470
2471   if (!is_sm_addr_only (flags))
2472     {
2473       m->local_port = l_port;
2474       m->external_port = e_port;
2475       m->proto = proto;
2476     }
2477
2478   if (is_sm_identity_nat (flags))
2479     {
2480       pool_get (m->locals, local);
2481
2482       local->vrf_id = vrf_id;
2483       local->fib_index = fib_index;
2484     }
2485   else
2486     {
2487       m->vrf_id = vrf_id;
2488       m->fib_index = fib_index;
2489     }
2490
2491   init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
2492                m - nm->static_mappings);
2493   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2494
2495   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
2496                m - nm->static_mappings);
2497   clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 1);
2498
2499   if (nm->num_workers > 1)
2500     {
2501       // store worker index for this record
2502       ip4_header_t ip = {
2503         .src_address = m->local_addr,
2504       };
2505       worker_index = nat44_ei_get_in2out_worker_index (&ip, m->fib_index, 0);
2506       vec_add1 (m->workers, worker_index);
2507     }
2508   else
2509     {
2510       worker_index = nm->num_workers;
2511     }
2512   delete_matching_dynamic_sessions (m, worker_index);
2513
2514   if (is_sm_addr_only (flags))
2515     {
2516       nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 1);
2517     }
2518
2519   return 0;
2520 }
2521
2522 static int
2523 nat44_ei_del_static_mapping_internal (ip4_address_t l_addr,
2524                                       ip4_address_t e_addr, u16 l_port,
2525                                       u16 e_port, nat_protocol_t proto,
2526                                       u32 vrf_id, u32 sw_if_index, u32 flags)
2527 {
2528   nat44_ei_main_per_thread_data_t *tnm;
2529   nat44_ei_main_t *nm = &nat44_ei_main;
2530   clib_bihash_kv_8_8_t kv, value;
2531   nat44_ei_lb_addr_port_t *local;
2532   nat44_ei_static_mapping_t *m;
2533   u32 fib_index = ~0;
2534   nat44_ei_user_key_t u_key;
2535
2536   fail_if_disabled ();
2537
2538   if (is_sm_addr_only (flags))
2539     {
2540       e_port = l_port = proto = 0;
2541     }
2542
2543   if (is_sm_identity_nat (flags))
2544     {
2545       l_port = e_port;
2546       l_addr.as_u32 = e_addr.as_u32;
2547     }
2548
2549   // fib index 0
2550   init_nat_k (&kv, e_addr, e_port, 0, proto);
2551
2552   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2553     {
2554       if (is_sm_switch_address (flags))
2555         {
2556           return 0;
2557         }
2558       return VNET_API_ERROR_NO_SUCH_ENTRY;
2559     }
2560
2561   m = pool_elt_at_index (nm->static_mappings, value.value);
2562
2563   if (is_sm_identity_nat (flags))
2564     {
2565       u8 found = 0;
2566
2567       if (vrf_id == ~0)
2568         {
2569           vrf_id = nm->inside_vrf_id;
2570         }
2571
2572       pool_foreach (local, m->locals)
2573         {
2574           if (local->vrf_id == vrf_id)
2575             {
2576               local = pool_elt_at_index (m->locals, local - m->locals);
2577               fib_index = local->fib_index;
2578               pool_put (m->locals, local);
2579               found = 1;
2580             }
2581         }
2582       if (!found)
2583         {
2584           return VNET_API_ERROR_NO_SUCH_ENTRY;
2585         }
2586     }
2587   else
2588     {
2589       fib_index = m->fib_index;
2590     }
2591
2592   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2593     {
2594       if (nat44_ei_free_port (e_addr, e_port, proto))
2595         {
2596           return VNET_API_ERROR_INVALID_VALUE;
2597         }
2598     }
2599
2600   init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2601   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 0);
2602
2603   if (!nm->static_mapping_only || nm->static_mapping_connection_tracking)
2604     {
2605       // delete sessions for static mapping
2606       if (nm->num_workers > 1)
2607         tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2608       else
2609         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2610
2611       u_key.addr = m->local_addr;
2612       u_key.fib_index = fib_index;
2613       kv.key = u_key.as_u64;
2614       nat44_ei_static_mapping_del_sessions (
2615         nm, tnm, u_key, is_sm_addr_only (flags), e_addr, e_port);
2616     }
2617
2618   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2619
2620   if (!pool_elts (m->locals))
2621     {
2622       // this is last record remove all required stuff
2623       // fib_index 0
2624       init_nat_k (&kv, e_addr, e_port, 0, proto);
2625       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 0);
2626
2627       vec_free (m->tag);
2628       vec_free (m->workers);
2629       pool_put (nm->static_mappings, m);
2630
2631       if (is_sm_addr_only (flags) && !is_sm_identity_nat (flags))
2632         {
2633           nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 0);
2634         }
2635     }
2636
2637   return 0;
2638 }
2639
2640 int
2641 nat44_ei_static_mapping_match (ip4_address_t match_addr, u16 match_port,
2642                                u32 match_fib_index,
2643                                nat_protocol_t match_protocol,
2644                                ip4_address_t *mapping_addr, u16 *mapping_port,
2645                                u32 *mapping_fib_index, u8 by_external,
2646                                u8 *is_addr_only, u8 *is_identity_nat)
2647 {
2648   nat44_ei_main_t *nm = &nat44_ei_main;
2649   clib_bihash_kv_8_8_t kv, value;
2650   nat44_ei_static_mapping_t *m;
2651   u16 port;
2652
2653   if (by_external)
2654     {
2655       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2656       if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2657                                   &value))
2658         {
2659           /* Try address only mapping */
2660           init_nat_k (&kv, match_addr, 0, 0, 0);
2661           if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2662                                       &value))
2663             return 1;
2664         }
2665       m = pool_elt_at_index (nm->static_mappings, value.value);
2666
2667       *mapping_fib_index = m->fib_index;
2668       *mapping_addr = m->local_addr;
2669       port = m->local_port;
2670     }
2671   else
2672     {
2673       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2674                   match_protocol);
2675       if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2676         {
2677           /* Try address only mapping */
2678           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2679           if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
2680                                       &value))
2681             return 1;
2682         }
2683       m = pool_elt_at_index (nm->static_mappings, value.value);
2684
2685       *mapping_fib_index = nm->outside_fib_index;
2686       *mapping_addr = m->external_addr;
2687       port = m->external_port;
2688     }
2689
2690   /* Address only mapping doesn't change port */
2691   if (is_sm_addr_only (m->flags))
2692     *mapping_port = match_port;
2693   else
2694     *mapping_port = port;
2695
2696   if (PREDICT_FALSE (is_addr_only != 0))
2697     *is_addr_only = is_sm_addr_only (m->flags);
2698
2699   if (PREDICT_FALSE (is_identity_nat != 0))
2700     *is_identity_nat = is_sm_identity_nat (m->flags);
2701
2702   return 0;
2703 }
2704
2705 static void
2706 nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm)
2707 {
2708   pool_free (tnm->list_pool);
2709   pool_free (tnm->lru_pool);
2710   pool_free (tnm->sessions);
2711   pool_free (tnm->users);
2712
2713   clib_bihash_free_8_8 (&tnm->user_hash);
2714 }
2715
2716 u8 *
2717 format_nat44_ei_key (u8 *s, va_list *args)
2718 {
2719   u64 key = va_arg (*args, u64);
2720
2721   ip4_address_t addr;
2722   u16 port;
2723   nat_protocol_t protocol;
2724   u32 fib_index;
2725
2726   split_nat_key (key, &addr, &port, &fib_index, &protocol);
2727
2728   s = format (s, "%U proto %U port %d fib %d", format_ip4_address, &addr,
2729               format_nat_protocol, protocol, clib_net_to_host_u16 (port),
2730               fib_index);
2731   return s;
2732 }
2733
2734 u8 *
2735 format_nat44_ei_user_kvp (u8 *s, va_list *args)
2736 {
2737   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2738   nat44_ei_user_key_t k;
2739
2740   k.as_u64 = v->key;
2741
2742   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2743               k.fib_index, v->value);
2744
2745   return s;
2746 }
2747
2748 u8 *
2749 format_nat44_ei_session_kvp (u8 *s, va_list *args)
2750 {
2751   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2752
2753   s = format (s, "%U thread-index %llu session-index %llu",
2754               format_nat44_ei_key, v->key, nat_value_get_thread_index (v),
2755               nat_value_get_session_index (v));
2756
2757   return s;
2758 }
2759
2760 u8 *
2761 format_nat44_ei_static_mapping_kvp (u8 *s, va_list *args)
2762 {
2763   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2764
2765   s = format (s, "%U static-mapping-index %llu", format_nat44_ei_key, v->key,
2766               v->value);
2767
2768   return s;
2769 }
2770
2771 static void
2772 nat44_ei_worker_db_init (nat44_ei_main_per_thread_data_t *tnm,
2773                          u32 translations, u32 translation_buckets,
2774                          u32 user_buckets)
2775 {
2776   dlist_elt_t *head;
2777
2778   pool_alloc (tnm->list_pool, translations);
2779   pool_alloc (tnm->lru_pool, translations);
2780   pool_alloc (tnm->sessions, translations);
2781
2782   clib_bihash_init_8_8 (&tnm->user_hash, "users", user_buckets, 0);
2783
2784   clib_bihash_set_kvp_format_fn_8_8 (&tnm->user_hash,
2785                                      format_nat44_ei_user_kvp);
2786
2787   pool_get (tnm->lru_pool, head);
2788   tnm->tcp_trans_lru_head_index = head - tnm->lru_pool;
2789   clib_dlist_init (tnm->lru_pool, tnm->tcp_trans_lru_head_index);
2790
2791   pool_get (tnm->lru_pool, head);
2792   tnm->tcp_estab_lru_head_index = head - tnm->lru_pool;
2793   clib_dlist_init (tnm->lru_pool, tnm->tcp_estab_lru_head_index);
2794
2795   pool_get (tnm->lru_pool, head);
2796   tnm->udp_lru_head_index = head - tnm->lru_pool;
2797   clib_dlist_init (tnm->lru_pool, tnm->udp_lru_head_index);
2798
2799   pool_get (tnm->lru_pool, head);
2800   tnm->icmp_lru_head_index = head - tnm->lru_pool;
2801   clib_dlist_init (tnm->lru_pool, tnm->icmp_lru_head_index);
2802
2803   pool_get (tnm->lru_pool, head);
2804   tnm->unk_proto_lru_head_index = head - tnm->lru_pool;
2805   clib_dlist_init (tnm->lru_pool, tnm->unk_proto_lru_head_index);
2806 }
2807
2808 static void
2809 nat44_ei_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
2810 {
2811   nat44_ei_main_t *nm = &nat44_ei_main;
2812   nat44_ei_main_per_thread_data_t *tnm;
2813
2814   u32 static_mapping_buckets = 1024;
2815   u32 static_mapping_memory_size = 64 << 20;
2816
2817   clib_bihash_init_8_8 (&nm->static_mapping_by_local,
2818                         "static_mapping_by_local", static_mapping_buckets,
2819                         static_mapping_memory_size);
2820   clib_bihash_init_8_8 (&nm->static_mapping_by_external,
2821                         "static_mapping_by_external", static_mapping_buckets,
2822                         static_mapping_memory_size);
2823   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_local,
2824                                      format_nat44_ei_static_mapping_kvp);
2825   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_external,
2826                                      format_nat44_ei_static_mapping_kvp);
2827
2828   if (nm->pat)
2829     {
2830       clib_bihash_init_8_8 (&nm->in2out, "in2out", translation_buckets, 0);
2831       clib_bihash_init_8_8 (&nm->out2in, "out2in", translation_buckets, 0);
2832       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2833                                          format_nat44_ei_session_kvp);
2834       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2835                                          format_nat44_ei_session_kvp);
2836       vec_foreach (tnm, nm->per_thread_data)
2837         {
2838           nat44_ei_worker_db_init (tnm, translations, translation_buckets,
2839                                    user_buckets);
2840         }
2841     }
2842 }
2843
2844 void
2845 nat44_ei_sessions_clear ()
2846 {
2847   nat44_ei_main_t *nm = &nat44_ei_main;
2848   nat44_ei_main_per_thread_data_t *tnm;
2849
2850   if (nm->pat)
2851     {
2852       clib_bihash_free_8_8 (&nm->in2out);
2853       clib_bihash_free_8_8 (&nm->out2in);
2854       clib_bihash_init_8_8 (&nm->in2out, "in2out", nm->translation_buckets, 0);
2855       clib_bihash_init_8_8 (&nm->out2in, "out2in", nm->translation_buckets, 0);
2856       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2857                                          format_nat44_ei_session_kvp);
2858       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2859                                          format_nat44_ei_session_kvp);
2860       vec_foreach (tnm, nm->per_thread_data)
2861         {
2862           nat44_ei_worker_db_free (tnm);
2863           nat44_ei_worker_db_init (tnm, nm->translations,
2864                                    nm->translation_buckets, nm->user_buckets);
2865         }
2866     }
2867
2868   vlib_zero_simple_counter (&nm->total_users, 0);
2869   vlib_zero_simple_counter (&nm->total_sessions, 0);
2870   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
2871 }
2872
2873 static void
2874 nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque, u32 sw_if_index,
2875                              u32 new_fib_index, u32 old_fib_index)
2876 {
2877   nat44_ei_main_t *nm = &nat44_ei_main;
2878   nat44_ei_outside_fib_t *outside_fib;
2879   nat44_ei_interface_t *i;
2880   u8 is_add = 1;
2881   u8 match = 0;
2882
2883   if (!nm->enabled || (new_fib_index == old_fib_index) ||
2884       (!vec_len (nm->outside_fibs)))
2885     {
2886       return;
2887     }
2888
2889   pool_foreach (i, nm->interfaces)
2890     {
2891       if (i->sw_if_index == sw_if_index)
2892         {
2893           if (!(nat44_ei_interface_is_outside (i)))
2894             return;
2895           match = 1;
2896         }
2897     }
2898
2899   pool_foreach (i, nm->output_feature_interfaces)
2900     {
2901       if (i->sw_if_index == sw_if_index)
2902         {
2903           if (!(nat44_ei_interface_is_outside (i)))
2904             return;
2905           match = 1;
2906         }
2907     }
2908
2909   if (!match)
2910     return;
2911
2912   vec_foreach (outside_fib, nm->outside_fibs)
2913     {
2914       if (outside_fib->fib_index == old_fib_index)
2915         {
2916           outside_fib->refcount--;
2917           if (!outside_fib->refcount)
2918             vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
2919           break;
2920         }
2921     }
2922
2923   vec_foreach (outside_fib, nm->outside_fibs)
2924     {
2925       if (outside_fib->fib_index == new_fib_index)
2926         {
2927           outside_fib->refcount++;
2928           is_add = 0;
2929           break;
2930         }
2931     }
2932
2933   if (is_add)
2934     {
2935       vec_add2 (nm->outside_fibs, outside_fib, 1);
2936       outside_fib->refcount = 1;
2937       outside_fib->fib_index = new_fib_index;
2938     }
2939 }
2940
2941 int
2942 nat44_ei_add_address (ip4_address_t *addr, u32 vrf_id)
2943 {
2944   nat44_ei_main_t *nm = &nat44_ei_main;
2945   vlib_thread_main_t *tm = vlib_get_thread_main ();
2946   nat44_ei_address_t *ap;
2947
2948   fail_if_disabled ();
2949
2950   /* Check if address already exists */
2951   vec_foreach (ap, nm->addresses)
2952     {
2953       if (ap->addr.as_u32 == addr->as_u32)
2954         {
2955           nat44_ei_log_err ("address exist");
2956           return VNET_API_ERROR_VALUE_EXIST;
2957         }
2958     }
2959
2960   vec_add2 (nm->addresses, ap, 1);
2961
2962   ap->fib_index = ~0;
2963   ap->addr = *addr;
2964
2965   if (vrf_id != ~0)
2966     {
2967       ap->fib_index = fib_table_find_or_create_and_lock (
2968         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2969     }
2970
2971   nat_protocol_t proto;
2972   for (proto = 0; proto < NAT_N_PROTOCOLS; ++proto)
2973     {
2974       ap->busy_port_bitmap[proto] = 0;
2975       ap->busy_ports[proto] = 0;
2976       ap->busy_ports_per_thread[proto] = 0;
2977       vec_validate_init_empty (ap->busy_ports_per_thread[proto],
2978                                tm->n_vlib_mains - 1, 0);
2979     }
2980
2981     nat44_ei_add_del_addr_to_fib_foreach_out_if (addr, 1);
2982
2983   return 0;
2984 }
2985
2986 int
2987 nat44_ei_del_address (ip4_address_t addr, u8 delete_sm)
2988 {
2989   nat44_ei_main_t *nm = &nat44_ei_main;
2990   nat44_ei_address_t *a = 0;
2991   nat44_ei_session_t *ses;
2992   u32 *ses_to_be_removed = 0, *ses_index;
2993   nat44_ei_main_per_thread_data_t *tnm;
2994   nat44_ei_static_mapping_t *m;
2995   int j;
2996
2997   fail_if_disabled ();
2998
2999   /* Find SNAT address */
3000   for (j = 0; j < vec_len (nm->addresses); j++)
3001     {
3002       if (nm->addresses[j].addr.as_u32 == addr.as_u32)
3003         {
3004           a = nm->addresses + j;
3005           break;
3006         }
3007     }
3008   if (!a)
3009     {
3010       nat44_ei_log_err ("no such address");
3011       return VNET_API_ERROR_NO_SUCH_ENTRY;
3012     }
3013
3014   if (delete_sm)
3015     {
3016       pool_foreach (m, nm->static_mappings)
3017         {
3018           if (m->external_addr.as_u32 == addr.as_u32)
3019             nat44_ei_del_static_mapping_internal (
3020               m->local_addr, m->external_addr, m->local_port, m->external_port,
3021               m->proto, m->vrf_id, ~0, m->flags);
3022         }
3023     }
3024   else
3025     {
3026       /* Check if address is used in some static mapping */
3027       if (nat44_ei_is_address_used_in_static_mapping (addr))
3028         {
3029           nat44_ei_log_err ("address used in static mapping");
3030           return VNET_API_ERROR_UNSPECIFIED;
3031         }
3032     }
3033
3034   /* Delete sessions using address */
3035   if (a->busy_ports[NAT_PROTOCOL_TCP] || a->busy_ports[NAT_PROTOCOL_UDP] ||
3036       a->busy_ports[NAT_PROTOCOL_ICMP])
3037     {
3038       vec_foreach (tnm, nm->per_thread_data)
3039         {
3040           pool_foreach (ses, tnm->sessions)
3041             {
3042               if (ses->out2in.addr.as_u32 == addr.as_u32)
3043                 {
3044                   nat44_ei_free_session_data (nm, ses,
3045                                               tnm - nm->per_thread_data, 0);
3046                   vec_add1 (ses_to_be_removed, ses - tnm->sessions);
3047                 }
3048             }
3049           vec_foreach (ses_index, ses_to_be_removed)
3050             {
3051               ses = pool_elt_at_index (tnm->sessions, ses_index[0]);
3052               nat44_ei_delete_session (nm, ses, tnm - nm->per_thread_data);
3053             }
3054           vec_free (ses_to_be_removed);
3055         }
3056     }
3057
3058   nat44_ei_add_del_addr_to_fib_foreach_out_if (&addr, 0);
3059
3060   if (a->fib_index != ~0)
3061     {
3062       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
3063     }
3064
3065   nat_protocol_t proto;
3066   for (proto = 0; proto < NAT_N_PROTOCOLS; ++proto)
3067     {
3068       vec_free (a->busy_ports_per_thread[proto]);
3069     }
3070
3071   vec_del1 (nm->addresses, j);
3072   return 0;
3073 }
3074
3075 int
3076 nat44_ei_add_interface_address (u32 sw_if_index)
3077 {
3078   nat44_ei_main_t *nm = &nat44_ei_main;
3079   ip4_main_t *ip4_main = nm->ip4_main;
3080   ip4_address_t *first_int_addr;
3081   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3082   int i;
3083
3084   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3085     {
3086       if (auto_add_sw_if_indices[i] == sw_if_index)
3087         {
3088           return VNET_API_ERROR_VALUE_EXIST;
3089         }
3090     }
3091
3092   /* add to the auto-address list */
3093   vec_add1 (nm->auto_add_sw_if_indices, sw_if_index);
3094
3095   // if the address is already bound - or static - add it now
3096   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3097   if (first_int_addr)
3098     {
3099       (void) nat44_ei_add_address (first_int_addr, ~0);
3100     }
3101
3102   return 0;
3103 }
3104
3105 int
3106 nat44_ei_del_interface_address (u32 sw_if_index)
3107 {
3108   nat44_ei_main_t *nm = &nat44_ei_main;
3109   ip4_main_t *ip4_main = nm->ip4_main;
3110   ip4_address_t *first_int_addr;
3111   nat44_ei_static_map_resolve_t *rp;
3112   u32 *indices_to_delete = 0;
3113   int i, j;
3114   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3115
3116   fail_if_disabled ();
3117
3118   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3119
3120   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3121     {
3122       if (auto_add_sw_if_indices[i] == sw_if_index)
3123         {
3124           first_int_addr =
3125             ip4_interface_first_address (ip4_main, sw_if_index, 0);
3126           if (first_int_addr)
3127             {
3128               (void) nat44_ei_del_address (first_int_addr[0], 1);
3129             }
3130           else
3131             {
3132               for (j = 0; j < vec_len (nm->to_resolve); j++)
3133                 {
3134                   rp = nm->to_resolve + j;
3135                   if (rp->sw_if_index == sw_if_index)
3136                     {
3137                       vec_add1 (indices_to_delete, j);
3138                     }
3139                 }
3140               if (vec_len (indices_to_delete))
3141                 {
3142                   for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3143                     {
3144                       vec_del1 (nm->to_resolve, j);
3145                     }
3146                   vec_free (indices_to_delete);
3147                 }
3148             }
3149
3150           vec_del1 (nm->auto_add_sw_if_indices, i);
3151           return 0;
3152         }
3153     }
3154   return VNET_API_ERROR_NO_SUCH_ENTRY;
3155 }
3156
3157 static_always_inline int
3158 is_sw_if_index_reg_for_auto_resolve (u32 *sw_if_indices, u32 sw_if_index)
3159 {
3160   u32 *i;
3161   vec_foreach (i, sw_if_indices)
3162     {
3163       if (*i == sw_if_index)
3164         {
3165           return 1;
3166         }
3167     }
3168   return 0;
3169 }
3170
3171 static void
3172 nat44_ei_ip4_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3173                                            u32 sw_if_index,
3174                                            ip4_address_t *address,
3175                                            u32 address_length,
3176                                            u32 if_address_index, u32 is_delete)
3177 {
3178   nat44_ei_main_t *nm = &nat44_ei_main;
3179   nat44_ei_static_map_resolve_t *rp;
3180   nat44_ei_address_t *addresses = nm->addresses;
3181   int rv, i;
3182
3183   if (!nm->enabled)
3184     {
3185       return;
3186     }
3187
3188   if (!is_sw_if_index_reg_for_auto_resolve (nm->auto_add_sw_if_indices,
3189                                             sw_if_index))
3190     {
3191       return;
3192     }
3193
3194   if (!is_delete)
3195     {
3196       /* Don't trip over lease renewal, static config */
3197       for (i = 0; i < vec_len (addresses); i++)
3198         {
3199           if (addresses[i].addr.as_u32 == address->as_u32)
3200             {
3201               return;
3202             }
3203         }
3204
3205       (void) nat44_ei_add_address (address, ~0);
3206
3207       /* Scan static map resolution vector */
3208       for (i = 0; i < vec_len (nm->to_resolve); i++)
3209         {
3210           rp = nm->to_resolve + i;
3211           if (is_sm_addr_only (rp->flags))
3212             {
3213               continue;
3214             }
3215           /* On this interface? */
3216           if (rp->sw_if_index == sw_if_index)
3217             {
3218               rv = nat44_ei_add_static_mapping_internal (
3219                 rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3220                 rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3221               if (rv)
3222                 {
3223                   nat_elog_notice_X1 (
3224                     nm, "add_static_mapping_internal returned %d", "i4", rv);
3225                 }
3226             }
3227         }
3228     }
3229   else
3230     {
3231       // remove all static mapping records
3232       (void) nat44_ei_del_address (address[0], 1);
3233     }
3234 }
3235
3236 int
3237 nat44_ei_set_frame_queue_nelts (u32 frame_queue_nelts)
3238 {
3239   fail_if_enabled ();
3240   nat44_ei_main_t *nm = &nat44_ei_main;
3241   nm->frame_queue_nelts = frame_queue_nelts;
3242   return 0;
3243 }
3244
3245 static void
3246 nat44_ei_ip4_add_del_addr_only_sm_cb (ip4_main_t *im, uword opaque,
3247                                       u32 sw_if_index, ip4_address_t *address,
3248                                       u32 address_length, u32 if_address_index,
3249                                       u32 is_delete)
3250 {
3251   nat44_ei_main_t *nm = &nat44_ei_main;
3252   nat44_ei_static_map_resolve_t *rp;
3253   nat44_ei_static_mapping_t *m;
3254   clib_bihash_kv_8_8_t kv, value;
3255   int i, rv = 0, match = 0;
3256
3257   if (!nm->enabled)
3258     {
3259       return;
3260     }
3261
3262   for (i = 0; i < vec_len (nm->to_resolve); i++)
3263     {
3264       rp = nm->to_resolve + i;
3265
3266       if (is_sm_addr_only (rp->flags) && rp->sw_if_index == sw_if_index)
3267         {
3268           match = 1;
3269           break;
3270         }
3271     }
3272
3273   if (!match)
3274     {
3275       return;
3276     }
3277
3278   init_nat_k (&kv, *address, is_sm_addr_only (rp->flags) ? 0 : rp->e_port,
3279               nm->outside_fib_index,
3280               is_sm_addr_only (rp->flags) ? 0 : rp->proto);
3281   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
3282     m = 0;
3283   else
3284     m = pool_elt_at_index (nm->static_mappings, value.value);
3285
3286   if (is_delete)
3287     {
3288       if (!m)
3289         return;
3290       rv = nat44_ei_del_static_mapping_internal (
3291         rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto, rp->vrf_id,
3292         ~0, rp->flags);
3293       if (rv)
3294         {
3295           nat_elog_notice_X1 (nm, "nat44_ei_del_static_mapping returned %d",
3296                               "i4", rv);
3297         }
3298     }
3299   else
3300     {
3301       if (m)
3302         return;
3303       rv = nat44_ei_add_static_mapping_internal (
3304         rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto, rp->vrf_id,
3305         ~0, rp->flags, rp->pool_addr, rp->tag);
3306
3307       if (rv)
3308         {
3309           nat_elog_notice_X1 (nm, "nat44_ei_add_static_mapping returned %d",
3310                               "i4", rv);
3311         }
3312     }
3313 }
3314
3315 static_always_inline uword
3316 nat44_ei_classify_inline_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
3317                              vlib_frame_t *frame)
3318 {
3319   u32 n_left_from, *from, *to_next;
3320   nat44_ei_classify_next_t next_index;
3321   nat44_ei_main_t *nm = &nat44_ei_main;
3322   nat44_ei_static_mapping_t *m;
3323   u32 next_in2out = 0, next_out2in = 0;
3324
3325   from = vlib_frame_vector_args (frame);
3326   n_left_from = frame->n_vectors;
3327   next_index = node->cached_next_index;
3328
3329   while (n_left_from > 0)
3330     {
3331       u32 n_left_to_next;
3332
3333       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3334
3335       while (n_left_from > 0 && n_left_to_next > 0)
3336         {
3337           u32 bi0;
3338           vlib_buffer_t *b0;
3339           u32 next0 = NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3340           ip4_header_t *ip0;
3341           nat44_ei_address_t *ap;
3342           clib_bihash_kv_8_8_t kv0, value0;
3343
3344           /* speculatively enqueue b0 to the current next frame */
3345           bi0 = from[0];
3346           to_next[0] = bi0;
3347           from += 1;
3348           to_next += 1;
3349           n_left_from -= 1;
3350           n_left_to_next -= 1;
3351
3352           b0 = vlib_get_buffer (vm, bi0);
3353           ip0 = vlib_buffer_get_current (b0);
3354
3355           vec_foreach (ap, nm->addresses)
3356             {
3357               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
3358                 {
3359                   next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3360                   goto enqueue0;
3361                 }
3362             }
3363
3364           if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
3365             {
3366               init_nat_k (&kv0, ip0->dst_address, 0, 0, 0);
3367               /* try to classify the fragment based on IP header alone */
3368               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3369                                            &kv0, &value0))
3370                 {
3371                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3372                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3373                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3374                   goto enqueue0;
3375                 }
3376               init_nat_k (&kv0, ip0->dst_address,
3377                           vnet_buffer (b0)->ip.reass.l4_dst_port, 0,
3378                           ip_proto_to_nat_proto (ip0->protocol));
3379               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3380                                            &kv0, &value0))
3381                 {
3382                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3383                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3384                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3385                 }
3386             }
3387
3388         enqueue0:
3389           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
3390                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
3391             {
3392               nat44_ei_classify_trace_t *t =
3393                 vlib_add_trace (vm, node, b0, sizeof (*t));
3394               t->cached = 0;
3395               t->next_in2out = next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
3396             }
3397
3398           next_in2out += next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3399           next_out2in += next0 == NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3400
3401           /* verify speculative enqueue, maybe switch current next frame */
3402           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
3403                                            n_left_to_next, bi0, next0);
3404         }
3405
3406       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3407     }
3408
3409   vlib_node_increment_counter (
3410     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_IN2OUT, next_in2out);
3411   vlib_node_increment_counter (
3412     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_OUT2IN, next_out2in);
3413   return frame->n_vectors;
3414 }
3415
3416 VLIB_NODE_FN (nat44_ei_classify_node)
3417 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3418 {
3419   return nat44_ei_classify_inline_fn (vm, node, frame);
3420 }
3421
3422 VLIB_REGISTER_NODE (nat44_ei_classify_node) = {
3423   .name = "nat44-ei-classify",
3424   .vector_size = sizeof (u32),
3425   .format_trace = format_nat44_ei_classify_trace,
3426   .type = VLIB_NODE_TYPE_INTERNAL,
3427   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3428   .error_strings = nat44_ei_classify_error_strings,
3429   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3430   .next_nodes = {
3431     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out",
3432     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in",
3433     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3434   },
3435 };
3436
3437 VLIB_NODE_FN (nat44_ei_handoff_classify_node)
3438 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3439 {
3440   return nat44_ei_classify_inline_fn (vm, node, frame);
3441 }
3442
3443 VLIB_REGISTER_NODE (nat44_ei_handoff_classify_node) = {
3444   .name = "nat44-ei-handoff-classify",
3445   .vector_size = sizeof (u32),
3446   .format_trace = format_nat44_ei_classify_trace,
3447   .type = VLIB_NODE_TYPE_INTERNAL,
3448   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3449   .error_strings = nat44_ei_classify_error_strings,
3450   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3451   .next_nodes = {
3452     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out-worker-handoff",
3453     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in-worker-handoff",
3454     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3455   },
3456 };
3457
3458 /*
3459  * fd.io coding-style-patch-verification: ON
3460  *
3461  * Local Variables:
3462  * eval: (c-set-style "gnu")
3463  * End:
3464  */