src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/lib/ipfix_logging.h>
  25 #include <nat/lib/nat_syslog.h>
  26 #include <nat/nat_inlines.h>
  27 #include <nat/nat44/inlines.h>
  28 #include <nat/nat_affinity.h>
  29 #include <vnet/fib/fib_table.h>
  30 #include <vnet/fib/ip4_fib.h>
  31 #include <vnet/ip/reass/ip4_sv_reass.h>
  32 #include <vppinfra/bihash_16_8.h>
  33 #include <nat/nat44/ed_inlines.h>
  34 #include <vnet/ip/ip_table.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_ha.h>
  37 #include <nat/nat44-ei/nat44_ei.h>
  38
  39 #include <vpp/app/version.h>
  40 #include <nat/lib/nat_inlines.h>
  41
  42 snat_main_t snat_main;
  43
  44 #define skip_if_disabled()                                                    \
  45   do                                                                          \
  46     {                                                                         \
  47       snat_main_t *sm = &snat_main;                                           \
  48       if (PREDICT_FALSE (!sm->enabled))                                       \
  49         return;                                                               \
  50     }                                                                         \
  51   while (0)
  52
  53 #define fail_if_enabled()                                                     \
  54   do                                                                          \
  55     {                                                                         \
  56       snat_main_t *sm = &snat_main;                                           \
  57       if (PREDICT_FALSE (sm->enabled))                                        \
  58         {                                                                     \
  59           nat_log_err ("plugin enabled");                                     \
  60           return 1;                                                           \
  61         }                                                                     \
  62     }                                                                         \
  63   while (0)
  64
  65 #define fail_if_disabled()                                                    \
  66   do                                                                          \
  67     {                                                                         \
  68       snat_main_t *sm = &snat_main;                                           \
  69       if (PREDICT_FALSE (!sm->enabled))                                       \
  70         {                                                                     \
  71           nat_log_err ("plugin disabled");                                    \
  72           return 1;                                                           \
  73         }                                                                     \
  74     }                                                                         \
  75   while (0)
  76
  77 /* *INDENT-OFF* */
  78 /* Hook up input features */
  79 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat-pre-in2out",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  83                                "ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat-pre-out2in",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  89                                "ip4-dhcp-client-detect",
  90                                "ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-in2out-worker-handoff",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  96 };
  97 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-out2in-worker-handoff",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 101                                "ip4-dhcp-client-detect"),
 102 };
 103 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-in2out",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-out2in",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 112                                "ip4-dhcp-client-detect"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-classify",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 118 };
 119 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-ed-in2out",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 123 };
 124 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-ed-out2in",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 128                                "ip4-dhcp-client-detect"),
 129 };
 130 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 131   .arc_name = "ip4-unicast",
 132   .node_name = "nat44-ed-classify",
 133   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 134 };
 135 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 136   .arc_name = "ip4-unicast",
 137   .node_name = "nat44-handoff-classify",
 138   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 141   .arc_name = "ip4-unicast",
 142   .node_name = "nat44-in2out-fast",
 143   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 144 };
 145 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 146   .arc_name = "ip4-unicast",
 147   .node_name = "nat44-out2in-fast",
 148   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 149                                "ip4-dhcp-client-detect"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 152   .arc_name = "ip4-unicast",
 153   .node_name = "nat44-hairpin-dst",
 154   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 155 };
 156
 157 /* Hook up output features */
 158 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 159   .arc_name = "ip4-output",
 160   .node_name = "nat44-in2out-output",
 161   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 162 };
 163 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 164   .arc_name = "ip4-output",
 165   .node_name = "nat44-in2out-output-worker-handoff",
 166   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 167 };
 168 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 169   .arc_name = "ip4-output",
 170   .node_name = "nat44-hairpin-src",
 171   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 172 };
 173 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 174   .arc_name = "ip4-output",
 175   .node_name = "nat-pre-in2out-output",
 176   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 177   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 178 };
 179 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 180   .arc_name = "ip4-output",
 181   .node_name = "nat44-ed-in2out-output",
 182   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 183   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 184 };
 185
 186 /* Hook up ip4-local features */
 187 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 188 {
 189   .arc_name = "ip4-local",
 190   .node_name = "nat44-hairpinning",
 191   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 192 };
 193
 194 VLIB_PLUGIN_REGISTER () = {
 195     .version = VPP_BUILD_VER,
 196     .description = "Network Address Translation (NAT)",
 197 };
 198 /* *INDENT-ON* */
 199
 200 static void nat44_ed_db_init (u32 translations, u32 translation_buckets,
 201                               u32 user_buckets);
 202
 203 static void nat44_ed_db_free ();
 204
 205 static u32
 206 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
 207                                u32 rx_fib_index, u8 is_output);
 208
 209 static u32
 210 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
 211                                u8 is_output);
 212
 213 u32 nat_calc_bihash_buckets (u32 n_elts);
 214
 215 u8 *
 216 format_session_kvp (u8 * s, va_list * args)
 217 {
 218   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 219
 220   s = format (s, "%U thread-index %llu session-index %llu", format_snat_key,
 221               v->key, nat_value_get_thread_index (v),
 222               nat_value_get_session_index (v));
 223
 224   return s;
 225 }
 226
 227 u8 *
 228 format_static_mapping_kvp (u8 * s, va_list * args)
 229 {
 230   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 231
 232   s = format (s, "%U static-mapping-index %llu",
 233               format_snat_key, v->key, v->value);
 234
 235   return s;
 236 }
 237
 238 u8 *
 239 format_user_kvp (u8 * s, va_list * args)
 240 {
 241   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 242   snat_user_key_t k;
 243
 244   k.as_u64 = v->key;
 245
 246   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
 247               k.fib_index, v->value);
 248
 249   return s;
 250 }
 251
 252 u8 *
 253 format_ed_session_kvp (u8 * s, va_list * args)
 254 {
 255   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 256
 257   u8 proto;
 258   u16 r_port, l_port;
 259   ip4_address_t l_addr, r_addr;
 260   u32 fib_index;
 261
 262   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 263   s = format (s,
 264               "local %U:%d remote %U:%d proto %U fib %d thread-index %u "
 265               "session-index %u",
 266               format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 267               format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 268               format_ip_protocol, proto, fib_index,
 269               ed_value_get_thread_index (v), ed_value_get_session_index (v));
 270
 271   return s;
 272 }
 273
 274 void
 275 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 276                        u8 is_ha)
 277 {
 278   clib_bihash_kv_8_8_t kv;
 279
 280   if (is_ed_session (s))
 281     {
 282       per_vrf_sessions_unregister_session (s, thread_index);
 283
 284       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 285         nat_elog_warn ("flow hash del failed");
 286
 287       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 288         nat_elog_warn ("flow hash del failed");
 289     }
 290
 291   if (is_fwd_bypass_session (s))
 292     {
 293       return;
 294     }
 295
 296   /* session lookup tables */
 297   if (is_ed_session (s))
 298     {
 299       if (is_affinity_sessions (s))
 300         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 301                              s->nat_proto, s->out2in.port);
 302
 303       if (!is_ha)
 304         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 305                                &s->in2out.addr, s->in2out.port,
 306                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 307                                &s->out2in.addr, s->out2in.port,
 308                                &s->ext_host_addr, s->ext_host_port,
 309                                s->nat_proto, is_twice_nat_session (s));
 310     }
 311   else
 312     {
 313       init_nat_i2o_k (&kv, s);
 314       if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
 315         nat_elog_warn ("in2out key del failed");
 316       init_nat_o2i_k (&kv, s);
 317       if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
 318         nat_elog_warn ("out2in key del failed");
 319
 320       if (!is_ha)
 321         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 322                                  &s->in2out.addr, s->in2out.port,
 323                                  &s->out2in.addr, s->out2in.port,
 324                                  s->nat_proto);
 325     }
 326
 327   if (snat_is_unk_proto_session (s))
 328     return;
 329
 330   if (!is_ha)
 331     {
 332       /* log NAT event */
 333       nat_ipfix_logging_nat44_ses_delete (thread_index,
 334                                           s->in2out.addr.as_u32,
 335                                           s->out2in.addr.as_u32,
 336                                           s->nat_proto,
 337                                           s->in2out.port,
 338                                           s->out2in.port,
 339                                           s->in2out.fib_index);
 340
 341       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 342                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 343                    thread_index);
 344     }
 345
 346   /* Twice NAT address and port for external host */
 347   if (is_twice_nat_session (s))
 348     {
 349       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 350                                           thread_index,
 351                                           &s->ext_host_nat_addr,
 352                                           s->ext_host_nat_port, s->nat_proto);
 353     }
 354
 355   if (snat_is_session_static (s))
 356     return;
 357
 358   snat_free_outside_address_and_port (sm->addresses, thread_index,
 359                                       &s->out2in.addr, s->out2in.port,
 360                                       s->nat_proto);
 361 }
 362
 363 snat_user_t *
 364 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 365                         u32 thread_index)
 366 {
 367   snat_user_t *u = 0;
 368   snat_user_key_t user_key;
 369   clib_bihash_kv_8_8_t kv, value;
 370   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 371   dlist_elt_t *per_user_list_head_elt;
 372
 373   user_key.addr.as_u32 = addr->as_u32;
 374   user_key.fib_index = fib_index;
 375   kv.key = user_key.as_u64;
 376
 377   /* Ever heard of the "user" = src ip4 address before? */
 378   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 379     {
 380       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 381         {
 382           vlib_increment_simple_counter (&sm->user_limit_reached,
 383                                          thread_index, 0, 1);
 384           nat_elog_warn ("maximum user limit reached");
 385           return NULL;
 386         }
 387       /* no, make a new one */
 388       pool_get (tsm->users, u);
 389       clib_memset (u, 0, sizeof (*u));
 390
 391       u->addr.as_u32 = addr->as_u32;
 392       u->fib_index = fib_index;
 393
 394       pool_get (tsm->list_pool, per_user_list_head_elt);
 395
 396       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 397         tsm->list_pool;
 398
 399       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 400
 401       kv.value = u - tsm->users;
 402
 403       /* add user */
 404       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 405         {
 406           nat_elog_warn ("user_hash key add failed");
 407           nat44_delete_user_with_no_session (sm, u, thread_index);
 408           return NULL;
 409         }
 410
 411       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 412                                pool_elts (tsm->users));
 413     }
 414   else
 415     {
 416       u = pool_elt_at_index (tsm->users, value.value);
 417     }
 418
 419   return u;
 420 }
 421
 422 // only NAT EI
 423 snat_session_t *
 424 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 425                               u32 thread_index, f64 now)
 426 {
 427   snat_session_t *s;
 428   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 429   u32 oldest_per_user_translation_list_index, session_index;
 430   dlist_elt_t *oldest_per_user_translation_list_elt;
 431   dlist_elt_t *per_user_translation_list_elt;
 432
 433   /* Over quota? Recycle the least recently used translation */
 434   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 435     {
 436       oldest_per_user_translation_list_index =
 437         clib_dlist_remove_head (tsm->list_pool,
 438                                 u->sessions_per_user_list_head_index);
 439
 440       ASSERT (oldest_per_user_translation_list_index != ~0);
 441
 442       /* Add it back to the end of the LRU list */
 443       clib_dlist_addtail (tsm->list_pool,
 444                           u->sessions_per_user_list_head_index,
 445                           oldest_per_user_translation_list_index);
 446       /* Get the list element */
 447       oldest_per_user_translation_list_elt =
 448         pool_elt_at_index (tsm->list_pool,
 449                            oldest_per_user_translation_list_index);
 450
 451       /* Get the session index from the list element */
 452       session_index = oldest_per_user_translation_list_elt->value;
 453
 454       /* Get the session */
 455       s = pool_elt_at_index (tsm->sessions, session_index);
 456
 457       // TODO: ONLY EI version should be called
 458       nat_free_session_data (sm, s, thread_index, 0);
 459       if (snat_is_session_static (s))
 460         u->nstaticsessions--;
 461       else
 462         u->nsessions--;
 463       s->flags = 0;
 464       s->total_bytes = 0;
 465       s->total_pkts = 0;
 466       s->state = 0;
 467       s->ext_host_addr.as_u32 = 0;
 468       s->ext_host_port = 0;
 469       s->ext_host_nat_addr.as_u32 = 0;
 470       s->ext_host_nat_port = 0;
 471     }
 472   else
 473     {
 474       pool_get (tsm->sessions, s);
 475       clib_memset (s, 0, sizeof (*s));
 476
 477       /* Create list elts */
 478       pool_get (tsm->list_pool, per_user_translation_list_elt);
 479       clib_dlist_init (tsm->list_pool,
 480                        per_user_translation_list_elt - tsm->list_pool);
 481
 482       per_user_translation_list_elt->value = s - tsm->sessions;
 483       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 484       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 485
 486       clib_dlist_addtail (tsm->list_pool,
 487                           s->per_user_list_head_index,
 488                           per_user_translation_list_elt - tsm->list_pool);
 489
 490       s->user_index = u - tsm->users;
 491       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 492                                pool_elts (tsm->sessions));
 493     }
 494
 495   s->ha_last_refreshed = now;
 496
 497   return s;
 498 }
 499
 500 void
 501 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 502                           int is_add)
 503 {
 504   snat_main_t *sm = &snat_main;
 505   fib_prefix_t prefix = {
 506     .fp_len = p_len,
 507     .fp_proto = FIB_PROTOCOL_IP4,
 508     .fp_addr = {
 509                 .ip4.as_u32 = addr->as_u32,
 510                 },
 511   };
 512   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 513
 514   if (is_add)
 515     fib_table_entry_update_one_path (fib_index,
 516                                      &prefix,
 517                                      sm->fib_src_low,
 518                                      (FIB_ENTRY_FLAG_CONNECTED |
 519                                       FIB_ENTRY_FLAG_LOCAL |
 520                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 521                                      DPO_PROTO_IP4,
 522                                      NULL,
 523                                      sw_if_index,
 524                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 525   else
 526     fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 527 }
 528
 529 int
 530 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 531                   u8 twice_nat)
 532 {
 533   snat_address_t *ap;
 534   snat_interface_t *i;
 535   vlib_thread_main_t *tm = vlib_get_thread_main ();
 536
 537   if (twice_nat && !sm->endpoint_dependent)
 538     {
 539       nat_log_err ("unsupported");
 540       return VNET_API_ERROR_UNSUPPORTED;
 541     }
 542
 543   /* Check if address already exists */
 544   /* *INDENT-OFF* */
 545   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 546     {
 547       if (ap->addr.as_u32 == addr->as_u32)
 548         {
 549           nat_log_err ("address exist");
 550           return VNET_API_ERROR_VALUE_EXIST;
 551         }
 552     }
 553   /* *INDENT-ON* */
 554
 555   if (twice_nat)
 556     vec_add2 (sm->twice_nat_addresses, ap, 1);
 557   else
 558     vec_add2 (sm->addresses, ap, 1);
 559
 560   ap->addr = *addr;
 561   if (vrf_id != ~0)
 562     ap->fib_index =
 563       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 564                                          sm->fib_src_low);
 565   else
 566     ap->fib_index = ~0;
 567
 568   /* *INDENT-OFF* */
 569   #define _(N, i, n, s) \
 570     clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 571     ap->busy_##n##_ports = 0; \
 572     ap->busy_##n##_ports_per_thread = 0;\
 573     vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 574     foreach_nat_protocol
 575   #undef _
 576   /* *INDENT-ON* */
 577
 578   if (twice_nat)
 579     return 0;
 580
 581   /* Add external address to FIB */
 582   /* *INDENT-OFF* */
 583   pool_foreach (i, sm->interfaces)
 584    {
 585     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 586       continue;
 587
 588     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 589     break;
 590   }
 591   pool_foreach (i, sm->output_feature_interfaces)
 592    {
 593     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 594       continue;
 595
 596     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 597     break;
 598   }
 599   /* *INDENT-ON* */
 600
 601   return 0;
 602 }
 603
 604 static int
 605 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 606 {
 607   snat_static_mapping_t *m;
 608   /* *INDENT-OFF* */
 609   pool_foreach (m, sm->static_mappings)
 610    {
 611       if (is_addr_only_static_mapping (m) ||
 612           is_out2in_only_static_mapping (m) ||
 613           is_identity_static_mapping (m))
 614         continue;
 615       if (m->external_addr.as_u32 == addr.as_u32)
 616         return 1;
 617   }
 618   /* *INDENT-ON* */
 619
 620   return 0;
 621 }
 622
 623 static void
 624 snat_add_static_mapping_when_resolved (snat_main_t *sm, ip4_address_t l_addr,
 625                                        u16 l_port, u32 sw_if_index, u16 e_port,
 626                                        u32 vrf_id, nat_protocol_t proto,
 627                                        int addr_only, u8 *tag, int twice_nat,
 628                                        int out2in_only, int identity_nat,
 629                                        ip4_address_t pool_addr, int exact)
 630 {
 631   snat_static_map_resolve_t *rp;
 632
 633   vec_add2 (sm->to_resolve, rp, 1);
 634   rp->l_addr.as_u32 = l_addr.as_u32;
 635   rp->l_port = l_port;
 636   rp->sw_if_index = sw_if_index;
 637   rp->e_port = e_port;
 638   rp->vrf_id = vrf_id;
 639   rp->proto = proto;
 640   rp->addr_only = addr_only;
 641   rp->twice_nat = twice_nat;
 642   rp->out2in_only = out2in_only;
 643   rp->identity_nat = identity_nat;
 644   rp->tag = vec_dup (tag);
 645   rp->pool_addr = pool_addr;
 646   rp->exact = exact;
 647 }
 648
 649 u32
 650 get_thread_idx_by_port (u16 e_port)
 651 {
 652   snat_main_t *sm = &snat_main;
 653   u32 thread_idx = sm->num_workers;
 654   if (sm->num_workers > 1)
 655     {
 656       thread_idx =
 657         sm->first_worker_index +
 658         sm->workers[(e_port - 1024) / sm->port_per_thread];
 659     }
 660   return thread_idx;
 661 }
 662
 663 void
 664 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 665                                     snat_main_per_thread_data_t * tsm,
 666                                     ip4_address_t l_addr,
 667                                     u16 l_port,
 668                                     u8 protocol,
 669                                     u32 fib_index, int addr_only,
 670                                     ip4_address_t e_addr, u16 e_port)
 671 {
 672   snat_session_t *s;
 673   u32 *indexes_to_free = NULL;
 674   /* *INDENT-OFF* */
 675   pool_foreach (s, tsm->sessions) {
 676     if (s->in2out.fib_index != fib_index ||
 677         s->in2out.addr.as_u32 != l_addr.as_u32)
 678       {
 679         continue;
 680       }
 681     if (!addr_only)
 682       {
 683         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 684             s->out2in.port != e_port ||
 685             s->in2out.port != l_port ||
 686             s->nat_proto != protocol)
 687           continue;
 688       }
 689
 690     if (is_lb_session (s))
 691       continue;
 692     if (!snat_is_session_static (s))
 693       continue;
 694     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 695     vec_add1 (indexes_to_free, s - tsm->sessions);
 696     if (!addr_only)
 697       break;
 698   }
 699   /* *INDENT-ON* */
 700   u32 *ses_index;
 701   vec_foreach (ses_index, indexes_to_free)
 702   {
 703     s = pool_elt_at_index (tsm->sessions, *ses_index);
 704     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 705   }
 706   vec_free (indexes_to_free);
 707 }
 708
 709 static_always_inline int
 710 nat44_ed_add_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 711                                  u16 l_port, u16 e_port, u32 vrf_id,
 712                                  int addr_only, u32 sw_if_index,
 713                                  nat_protocol_t proto, int is_add,
 714                                  twice_nat_type_t twice_nat, u8 out2in_only,
 715                                  u8 *tag, u8 identity_nat,
 716                                  ip4_address_t pool_addr, int exact)
 717 {
 718   snat_main_t *sm = &snat_main;
 719   snat_static_mapping_t *m;
 720   clib_bihash_kv_8_8_t kv, value;
 721   snat_address_t *a = 0;
 722   u32 fib_index = ~0;
 723   snat_interface_t *interface;
 724   int i;
 725   snat_main_per_thread_data_t *tsm;
 726   snat_user_key_t u_key;
 727   snat_user_t *u;
 728   dlist_elt_t *head, *elt;
 729   u32 elt_index, head_index;
 730   u32 ses_index;
 731   u64 user_index;
 732   snat_session_t *s;
 733   snat_static_map_resolve_t *rp, *rp_match = 0;
 734   nat44_lb_addr_port_t *local;
 735   u32 find = ~0;
 736
 737   if (!sm->endpoint_dependent)
 738     {
 739       if (twice_nat || out2in_only)
 740         return VNET_API_ERROR_UNSUPPORTED;
 741     }
 742
 743   /* If the external address is a specific interface address */
 744   if (sw_if_index != ~0)
 745     {
 746       ip4_address_t *first_int_addr;
 747
 748       for (i = 0; i < vec_len (sm->to_resolve); i++)
 749         {
 750           rp = sm->to_resolve + i;
 751           if (rp->sw_if_index != sw_if_index ||
 752               rp->l_addr.as_u32 != l_addr.as_u32 ||
 753               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 754             continue;
 755
 756           if (!addr_only)
 757             {
 758               if ((rp->l_port != l_port && rp->e_port != e_port)
 759                   || rp->proto != proto)
 760                 continue;
 761             }
 762
 763           rp_match = rp;
 764           break;
 765         }
 766
 767       /* Might be already set... */
 768       first_int_addr = ip4_interface_first_address
 769         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 770
 771       if (is_add)
 772         {
 773           if (rp_match)
 774             return VNET_API_ERROR_VALUE_EXIST;
 775
 776           snat_add_static_mapping_when_resolved (
 777             sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto, addr_only,
 778             tag, twice_nat, out2in_only, identity_nat, pool_addr, exact);
 779
 780           /* DHCP resolution required? */
 781           if (first_int_addr == 0)
 782             {
 783               return 0;
 784             }
 785           else
 786             {
 787               e_addr.as_u32 = first_int_addr->as_u32;
 788               /* Identity mapping? */
 789               if (l_addr.as_u32 == 0)
 790                 l_addr.as_u32 = e_addr.as_u32;
 791             }
 792         }
 793       else
 794         {
 795           if (!rp_match)
 796             return VNET_API_ERROR_NO_SUCH_ENTRY;
 797
 798           vec_del1 (sm->to_resolve, i);
 799
 800           if (first_int_addr)
 801             {
 802               e_addr.as_u32 = first_int_addr->as_u32;
 803               /* Identity mapping? */
 804               if (l_addr.as_u32 == 0)
 805                 l_addr.as_u32 = e_addr.as_u32;
 806             }
 807           else
 808             return 0;
 809         }
 810     }
 811
 812   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 813   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 814     m = 0;
 815   else
 816     m = pool_elt_at_index (sm->static_mappings, value.value);
 817
 818   if (is_add)
 819     {
 820       if (m)
 821         {
 822           if (is_identity_static_mapping (m))
 823             {
 824               /* *INDENT-OFF* */
 825               pool_foreach (local, m->locals)
 826                {
 827                 if (local->vrf_id == vrf_id)
 828                   return VNET_API_ERROR_VALUE_EXIST;
 829               }
 830               /* *INDENT-ON* */
 831               pool_get (m->locals, local);
 832               local->vrf_id = vrf_id;
 833               local->fib_index =
 834                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 835                                                    sm->fib_src_low);
 836               init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
 837                            m->proto, 0, m - sm->static_mappings);
 838               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 839               return 0;
 840             }
 841           else
 842             return VNET_API_ERROR_VALUE_EXIST;
 843         }
 844
 845       if (twice_nat && addr_only)
 846         return VNET_API_ERROR_UNSUPPORTED;
 847
 848       /* Convert VRF id to FIB index */
 849       if (vrf_id != ~0)
 850         fib_index =
 851           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 852                                              sm->fib_src_low);
 853       /* If not specified use inside VRF id from SNAT plugin startup config */
 854       else
 855         {
 856           fib_index = sm->inside_fib_index;
 857           vrf_id = sm->inside_vrf_id;
 858           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 859         }
 860
 861       if (!(out2in_only || identity_nat))
 862         {
 863           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
 864                       addr_only ? 0 : proto);
 865           if (!clib_bihash_search_8_8
 866               (&sm->static_mapping_by_local, &kv, &value))
 867             return VNET_API_ERROR_VALUE_EXIST;
 868         }
 869
 870       /* Find external address in allocated addresses and reserve port for
 871          address and port pair mapping when dynamic translations enabled */
 872       if (!(addr_only || sm->static_mapping_only || out2in_only))
 873         {
 874           for (i = 0; i < vec_len (sm->addresses); i++)
 875             {
 876               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 877                 {
 878                   a = sm->addresses + i;
 879                   /* External port must be unused */
 880                   switch (proto)
 881                     {
 882 #define _(N, j, n, s) \
 883                     case NAT_PROTOCOL_##N: \
 884                       if (a->busy_##n##_port_refcounts[e_port]) \
 885                         return VNET_API_ERROR_INVALID_VALUE; \
 886                       ++a->busy_##n##_port_refcounts[e_port]; \
 887                       if (e_port > 1024) \
 888                         { \
 889                           a->busy_##n##_ports++; \
 890                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 891                         } \
 892                       break;
 893                       foreach_nat_protocol
 894 #undef _
 895                     default:
 896                       nat_elog_info ("unknown protocol");
 897                       return VNET_API_ERROR_INVALID_VALUE_2;
 898                     }
 899                   break;
 900                 }
 901             }
 902           /* External address must be allocated */
 903           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 904             {
 905               if (sw_if_index != ~0)
 906                 {
 907                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 908                     {
 909                       rp = sm->to_resolve + i;
 910                       if (rp->addr_only)
 911                         continue;
 912                       if (rp->sw_if_index != sw_if_index &&
 913                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 914                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 915                           rp->e_port != e_port && rp->proto != proto)
 916                         continue;
 917
 918                       vec_del1 (sm->to_resolve, i);
 919                       break;
 920                     }
 921                 }
 922               return VNET_API_ERROR_NO_SUCH_ENTRY;
 923             }
 924         }
 925
 926       pool_get (sm->static_mappings, m);
 927       clib_memset (m, 0, sizeof (*m));
 928       m->tag = vec_dup (tag);
 929       m->local_addr = l_addr;
 930       m->external_addr = e_addr;
 931       m->twice_nat = twice_nat;
 932
 933       if (twice_nat == TWICE_NAT && exact)
 934         {
 935           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
 936           m->pool_addr = pool_addr;
 937         }
 938
 939       if (out2in_only)
 940         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 941       if (addr_only)
 942         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 943       if (identity_nat)
 944         {
 945           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 946           pool_get (m->locals, local);
 947           local->vrf_id = vrf_id;
 948           local->fib_index = fib_index;
 949         }
 950       else
 951         {
 952           m->vrf_id = vrf_id;
 953           m->fib_index = fib_index;
 954         }
 955       if (!addr_only)
 956         {
 957           m->local_port = l_port;
 958           m->external_port = e_port;
 959           m->proto = proto;
 960         }
 961
 962       if (sm->num_workers > 1)
 963         {
 964           ip4_header_t ip = {
 965             .src_address = m->local_addr,
 966           };
 967           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 968           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 969         }
 970       else
 971         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 972
 973       if (!out2in_only)
 974         {
 975           init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
 976                        0, m - sm->static_mappings);
 977           clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 978         }
 979
 980       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
 981                    m - sm->static_mappings);
 982       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 983
 984       /* Delete dynamic sessions matching local address (+ local port) */
 985       // TODO: based on type of NAT EI/ED
 986       if (!(sm->static_mapping_only))
 987         {
 988           u_key.addr = m->local_addr;
 989           u_key.fib_index = m->fib_index;
 990           kv.key = u_key.as_u64;
 991           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 992             {
 993               user_index = value.value;
 994               u = pool_elt_at_index (tsm->users, user_index);
 995               if (u->nsessions)
 996                 {
 997                   head_index = u->sessions_per_user_list_head_index;
 998                   head = pool_elt_at_index (tsm->list_pool, head_index);
 999                   elt_index = head->next;
1000                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1001                   ses_index = elt->value;
1002                   while (ses_index != ~0)
1003                     {
1004                       s = pool_elt_at_index (tsm->sessions, ses_index);
1005                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1006                       ses_index = elt->value;
1007
1008                       if (snat_is_session_static (s))
1009                         continue;
1010
1011                       if (!addr_only && s->in2out.port != m->local_port)
1012                         continue;
1013
1014                       nat_free_session_data (sm, s,
1015                                              tsm - sm->per_thread_data, 0);
1016                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1017
1018                       if (!addr_only && !sm->endpoint_dependent)
1019                         break;
1020                     }
1021                 }
1022             }
1023         }
1024     }
1025   else
1026     {
1027       if (!m)
1028         {
1029           if (sw_if_index != ~0)
1030             return 0;
1031           else
1032             return VNET_API_ERROR_NO_SUCH_ENTRY;
1033         }
1034
1035       if (identity_nat)
1036         {
1037           if (vrf_id == ~0)
1038             vrf_id = sm->inside_vrf_id;
1039
1040           /* *INDENT-OFF* */
1041           pool_foreach (local, m->locals)
1042            {
1043             if (local->vrf_id == vrf_id)
1044               find = local - m->locals;
1045           }
1046           /* *INDENT-ON* */
1047           if (find == ~0)
1048             return VNET_API_ERROR_NO_SUCH_ENTRY;
1049
1050           local = pool_elt_at_index (m->locals, find);
1051           fib_index = local->fib_index;
1052           pool_put (m->locals, local);
1053         }
1054       else
1055         fib_index = m->fib_index;
1056
1057       /* Free external address port */
1058       if (!(addr_only || sm->static_mapping_only || out2in_only))
1059         {
1060           for (i = 0; i < vec_len (sm->addresses); i++)
1061             {
1062               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1063                 {
1064                   a = sm->addresses + i;
1065                   switch (proto)
1066                     {
1067 #define _(N, j, n, s) \
1068                     case NAT_PROTOCOL_##N: \
1069                       --a->busy_##n##_port_refcounts[e_port]; \
1070                       if (e_port > 1024) \
1071                         { \
1072                           a->busy_##n##_ports--; \
1073                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1074                         } \
1075                       break;
1076                       foreach_nat_protocol
1077 #undef _
1078                     default:
1079                       nat_elog_info ("unknown protocol");
1080                       return VNET_API_ERROR_INVALID_VALUE_2;
1081                     }
1082                   break;
1083                 }
1084             }
1085         }
1086
1087       if (sm->num_workers > 1)
1088         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1089       else
1090         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1091
1092       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1093       if (!out2in_only)
1094         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1095
1096       /* Delete session(s) for static mapping if exist */
1097       if (!(sm->static_mapping_only) ||
1098           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1099         {
1100           if (sm->endpoint_dependent)
1101             {
1102               nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1103                                                   m->local_port, m->proto,
1104                                                   fib_index, addr_only,
1105                                                   e_addr, e_port);
1106             }
1107           else
1108             {
1109               u_key.addr = m->local_addr;
1110               u_key.fib_index = fib_index;
1111               kv.key = u_key.as_u64;
1112               nat44_ei_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1113                                                     e_addr, e_port);
1114             }
1115         }
1116
1117       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1118       if (pool_elts (m->locals))
1119         return 0;
1120
1121       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1122       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1123
1124       vec_free (m->tag);
1125       vec_free (m->workers);
1126       /* Delete static mapping from pool */
1127       pool_put (sm->static_mappings, m);
1128     }
1129
1130   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1131     return 0;
1132
1133   /* Add/delete external address to FIB */
1134   /* *INDENT-OFF* */
1135   pool_foreach (interface, sm->interfaces)
1136    {
1137     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1138       continue;
1139
1140     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1141     break;
1142   }
1143   pool_foreach (interface, sm->output_feature_interfaces)
1144    {
1145     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1146       continue;
1147
1148     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1149     break;
1150   }
1151   /* *INDENT-ON* */
1152
1153   return 0;
1154 }
1155
1156 int
1157 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
1158                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
1159                          u32 sw_if_index, nat_protocol_t proto, int is_add,
1160                          twice_nat_type_t twice_nat, u8 out2in_only, u8 *tag,
1161                          u8 identity_nat, ip4_address_t pool_addr, int exact)
1162 {
1163   snat_main_t *sm = &snat_main;
1164   int rv;
1165
1166   if (sm->endpoint_dependent)
1167     {
1168       rv = nat44_ed_add_del_static_mapping (
1169         l_addr, e_addr, l_port, e_port, vrf_id, addr_only, sw_if_index, proto,
1170         is_add, twice_nat, out2in_only, tag, identity_nat, pool_addr, exact);
1171     }
1172   else
1173     {
1174       rv = nat44_ei_add_del_static_mapping (
1175         l_addr, e_addr, l_port, e_port, proto, sw_if_index, vrf_id, addr_only,
1176         identity_nat, tag, is_add);
1177     }
1178   return rv;
1179 }
1180
1181 int
1182 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1183                                  nat_protocol_t proto,
1184                                  nat44_lb_addr_port_t * locals, u8 is_add,
1185                                  twice_nat_type_t twice_nat, u8 out2in_only,
1186                                  u8 * tag, u32 affinity)
1187 {
1188   snat_main_t *sm = &snat_main;
1189   snat_static_mapping_t *m;
1190   clib_bihash_kv_8_8_t kv, value;
1191   snat_address_t *a = 0;
1192   int i;
1193   nat44_lb_addr_port_t *local;
1194   snat_main_per_thread_data_t *tsm;
1195   snat_session_t *s;
1196   uword *bitmap = 0;
1197
1198   if (!sm->endpoint_dependent)
1199     return VNET_API_ERROR_UNSUPPORTED;
1200
1201   init_nat_k (&kv, e_addr, e_port, 0, proto);
1202   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1203     m = 0;
1204   else
1205     m = pool_elt_at_index (sm->static_mappings, value.value);
1206
1207   if (is_add)
1208     {
1209       if (m)
1210         return VNET_API_ERROR_VALUE_EXIST;
1211
1212       if (vec_len (locals) < 2)
1213         return VNET_API_ERROR_INVALID_VALUE;
1214
1215       /* Find external address in allocated addresses and reserve port for
1216          address and port pair mapping when dynamic translations enabled */
1217       if (!(sm->static_mapping_only || out2in_only))
1218         {
1219           for (i = 0; i < vec_len (sm->addresses); i++)
1220             {
1221               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1222                 {
1223                   a = sm->addresses + i;
1224                   /* External port must be unused */
1225                   switch (proto)
1226                     {
1227 #define _(N, j, n, s) \
1228                     case NAT_PROTOCOL_##N: \
1229                       if (a->busy_##n##_port_refcounts[e_port]) \
1230                         return VNET_API_ERROR_INVALID_VALUE; \
1231                       ++a->busy_##n##_port_refcounts[e_port]; \
1232                       if (e_port > 1024) \
1233                         { \
1234                           a->busy_##n##_ports++; \
1235                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1236                         } \
1237                       break;
1238                       foreach_nat_protocol
1239 #undef _
1240                     default:
1241                       nat_elog_info ("unknown protocol");
1242                       return VNET_API_ERROR_INVALID_VALUE_2;
1243                     }
1244                   break;
1245                 }
1246             }
1247           /* External address must be allocated */
1248           if (!a)
1249             return VNET_API_ERROR_NO_SUCH_ENTRY;
1250         }
1251
1252       pool_get (sm->static_mappings, m);
1253       clib_memset (m, 0, sizeof (*m));
1254       m->tag = vec_dup (tag);
1255       m->external_addr = e_addr;
1256       m->external_port = e_port;
1257       m->proto = proto;
1258       m->twice_nat = twice_nat;
1259       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1260       if (out2in_only)
1261         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1262       m->affinity = affinity;
1263
1264       if (affinity)
1265         m->affinity_per_service_list_head_index =
1266           nat_affinity_get_per_service_list_head_index ();
1267       else
1268         m->affinity_per_service_list_head_index = ~0;
1269
1270       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
1271                    m - sm->static_mappings);
1272       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1273         {
1274           nat_elog_err ("static_mapping_by_external key add failed");
1275           return VNET_API_ERROR_UNSPECIFIED;
1276         }
1277
1278       for (i = 0; i < vec_len (locals); i++)
1279         {
1280           locals[i].fib_index =
1281             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1282                                                locals[i].vrf_id,
1283                                                sm->fib_src_low);
1284           if (!out2in_only)
1285             {
1286               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1287                            locals[i].fib_index, m->proto, 0,
1288                            m - sm->static_mappings);
1289               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1290             }
1291           locals[i].prefix = (i == 0) ? locals[i].probability :
1292             (locals[i - 1].prefix + locals[i].probability);
1293           pool_get (m->locals, local);
1294           *local = locals[i];
1295           if (sm->num_workers > 1)
1296             {
1297               ip4_header_t ip = {
1298                 .src_address = locals[i].addr,
1299               };
1300               bitmap =
1301                 clib_bitmap_set (bitmap,
1302                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1303                                  1);
1304             }
1305         }
1306
1307       /* Assign workers */
1308       if (sm->num_workers > 1)
1309         {
1310           /* *INDENT-OFF* */
1311           clib_bitmap_foreach (i, bitmap)
1312              {
1313                vec_add1(m->workers, i);
1314             }
1315           /* *INDENT-ON* */
1316         }
1317     }
1318   else
1319     {
1320       if (!m)
1321         return VNET_API_ERROR_NO_SUCH_ENTRY;
1322
1323       if (!is_lb_static_mapping (m))
1324         return VNET_API_ERROR_INVALID_VALUE;
1325
1326       /* Free external address port */
1327       if (!(sm->static_mapping_only || out2in_only))
1328         {
1329           for (i = 0; i < vec_len (sm->addresses); i++)
1330             {
1331               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1332                 {
1333                   a = sm->addresses + i;
1334                   switch (proto)
1335                     {
1336 #define _(N, j, n, s) \
1337                     case NAT_PROTOCOL_##N: \
1338                       --a->busy_##n##_port_refcounts[e_port]; \
1339                       if (e_port > 1024) \
1340                         { \
1341                           a->busy_##n##_ports--; \
1342                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1343                         } \
1344                       break;
1345                       foreach_nat_protocol
1346 #undef _
1347                     default:
1348                       nat_elog_info ("unknown protocol");
1349                       return VNET_API_ERROR_INVALID_VALUE_2;
1350                     }
1351                   break;
1352                 }
1353             }
1354         }
1355
1356       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1357       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1358         {
1359           nat_elog_err ("static_mapping_by_external key del failed");
1360           return VNET_API_ERROR_UNSPECIFIED;
1361         }
1362
1363       /* *INDENT-OFF* */
1364       pool_foreach (local, m->locals)
1365       {
1366           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1367                             sm->fib_src_low);
1368           if (!out2in_only)
1369             {
1370 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1371               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1372                 {
1373                   nat_elog_err ("static_mapping_by_local key del failed");
1374                   return VNET_API_ERROR_UNSPECIFIED;
1375                 }
1376             }
1377
1378           if (sm->num_workers > 1)
1379             {
1380               ip4_header_t ip = {
1381                 .src_address = local->addr,
1382               };
1383               tsm = vec_elt_at_index (sm->per_thread_data,
1384                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1385             }
1386           else
1387             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1388
1389           /* Delete sessions */
1390           pool_foreach (s, tsm->sessions) {
1391             if (!(is_lb_session (s)))
1392               continue;
1393
1394             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1395                 s->in2out.port != local->port)
1396               continue;
1397
1398             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1399             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1400           }
1401       }
1402       /* *INDENT-ON* */
1403       if (m->affinity)
1404         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1405       pool_free (m->locals);
1406       vec_free (m->tag);
1407       vec_free (m->workers);
1408
1409       pool_put (sm->static_mappings, m);
1410     }
1411
1412   return 0;
1413 }
1414
1415 int
1416 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1417                                        ip4_address_t l_addr, u16 l_port,
1418                                        nat_protocol_t proto, u32 vrf_id,
1419                                        u8 probability, u8 is_add)
1420 {
1421   snat_main_t *sm = &snat_main;
1422   snat_static_mapping_t *m = 0;
1423   clib_bihash_kv_8_8_t kv, value;
1424   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1425   snat_main_per_thread_data_t *tsm;
1426   snat_session_t *s;
1427   u32 *locals = 0;
1428   uword *bitmap = 0;
1429   int i;
1430
1431   if (!sm->endpoint_dependent)
1432     return VNET_API_ERROR_FEATURE_DISABLED;
1433
1434   init_nat_k (&kv, e_addr, e_port, 0, proto);
1435   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1436     m = pool_elt_at_index (sm->static_mappings, value.value);
1437
1438   if (!m)
1439     return VNET_API_ERROR_NO_SUCH_ENTRY;
1440
1441   if (!is_lb_static_mapping (m))
1442     return VNET_API_ERROR_INVALID_VALUE;
1443
1444   /* *INDENT-OFF* */
1445   pool_foreach (local, m->locals)
1446    {
1447     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1448         (local->vrf_id == vrf_id))
1449       {
1450         match_local = local;
1451         break;
1452       }
1453   }
1454   /* *INDENT-ON* */
1455
1456   if (is_add)
1457     {
1458       if (match_local)
1459         return VNET_API_ERROR_VALUE_EXIST;
1460
1461       pool_get (m->locals, local);
1462       clib_memset (local, 0, sizeof (*local));
1463       local->addr.as_u32 = l_addr.as_u32;
1464       local->port = l_port;
1465       local->probability = probability;
1466       local->vrf_id = vrf_id;
1467       local->fib_index =
1468         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1469                                            sm->fib_src_low);
1470
1471       if (!is_out2in_only_static_mapping (m))
1472         {
1473           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto, 0,
1474                        m - sm->static_mappings);
1475           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1476             nat_elog_err ("static_mapping_by_local key add failed");
1477         }
1478     }
1479   else
1480     {
1481       if (!match_local)
1482         return VNET_API_ERROR_NO_SUCH_ENTRY;
1483
1484       if (pool_elts (m->locals) < 3)
1485         return VNET_API_ERROR_UNSPECIFIED;
1486
1487       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1488                         sm->fib_src_low);
1489
1490       if (!is_out2in_only_static_mapping (m))
1491         {
1492           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1493           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1494             nat_elog_err ("static_mapping_by_local key del failed");
1495         }
1496
1497       if (sm->num_workers > 1)
1498         {
1499           ip4_header_t ip = {
1500             .src_address = local->addr,
1501           };
1502           tsm = vec_elt_at_index (sm->per_thread_data,
1503                                   sm->worker_in2out_cb (&ip, m->fib_index,
1504                                                         0));
1505         }
1506       else
1507         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1508
1509       /* Delete sessions */
1510       /* *INDENT-OFF* */
1511       pool_foreach (s, tsm->sessions) {
1512         if (!(is_lb_session (s)))
1513           continue;
1514
1515         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1516             s->in2out.port != match_local->port)
1517           continue;
1518
1519         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1520         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1521       }
1522       /* *INDENT-ON* */
1523
1524       pool_put (m->locals, match_local);
1525     }
1526
1527   vec_free (m->workers);
1528
1529   /* *INDENT-OFF* */
1530   pool_foreach (local, m->locals)
1531    {
1532     vec_add1 (locals, local - m->locals);
1533     if (sm->num_workers > 1)
1534       {
1535         ip4_header_t ip;
1536         ip.src_address.as_u32 = local->addr.as_u32,
1537         bitmap = clib_bitmap_set (bitmap,
1538                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1539                                   1);
1540       }
1541   }
1542   /* *INDENT-ON* */
1543
1544   ASSERT (vec_len (locals) > 1);
1545
1546   local = pool_elt_at_index (m->locals, locals[0]);
1547   local->prefix = local->probability;
1548   for (i = 1; i < vec_len (locals); i++)
1549     {
1550       local = pool_elt_at_index (m->locals, locals[i]);
1551       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1552       local->prefix = local->probability + prev_local->prefix;
1553     }
1554
1555   /* Assign workers */
1556   if (sm->num_workers > 1)
1557     {
1558       /* *INDENT-OFF* */
1559       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1560       /* *INDENT-ON* */
1561     }
1562
1563   return 0;
1564 }
1565
1566 int
1567 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1568                   u8 twice_nat)
1569 {
1570   snat_address_t *a = 0;
1571   snat_session_t *ses;
1572   u32 *ses_to_be_removed = 0, *ses_index;
1573   snat_main_per_thread_data_t *tsm;
1574   snat_static_mapping_t *m;
1575   snat_interface_t *interface;
1576   int i;
1577   snat_address_t *addresses =
1578     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1579
1580   /* Find SNAT address */
1581   for (i = 0; i < vec_len (addresses); i++)
1582     {
1583       if (addresses[i].addr.as_u32 == addr.as_u32)
1584         {
1585           a = addresses + i;
1586           break;
1587         }
1588     }
1589   if (!a)
1590     {
1591       nat_log_err ("no such address");
1592       return VNET_API_ERROR_NO_SUCH_ENTRY;
1593     }
1594
1595   if (delete_sm)
1596     {
1597       ip4_address_t pool_addr = { 0 };
1598       /* *INDENT-OFF* */
1599       pool_foreach (m, sm->static_mappings)
1600        {
1601           if (m->external_addr.as_u32 == addr.as_u32)
1602             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1603                                             m->local_port, m->external_port,
1604                                             m->vrf_id,
1605                                             is_addr_only_static_mapping(m), ~0,
1606                                             m->proto, 0 /* is_add */,
1607                                             m->twice_nat,
1608                                             is_out2in_only_static_mapping(m),
1609                                             m->tag,
1610                                             is_identity_static_mapping(m),
1611                                             pool_addr, 0);
1612       }
1613       /* *INDENT-ON* */
1614     }
1615   else
1616     {
1617       /* Check if address is used in some static mapping */
1618       if (is_snat_address_used_in_static_mapping (sm, addr))
1619         {
1620           nat_log_err ("address used in static mapping");
1621           return VNET_API_ERROR_UNSPECIFIED;
1622         }
1623     }
1624
1625   if (a->fib_index != ~0)
1626     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1627
1628   /* Delete sessions using address */
1629   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1630     {
1631       vec_foreach (tsm, sm->per_thread_data)
1632       {
1633         /* *INDENT-OFF* */
1634         pool_foreach (ses, tsm->sessions)  {
1635           if (ses->out2in.addr.as_u32 == addr.as_u32)
1636             {
1637               nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1638               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1639             }
1640         }
1641         /* *INDENT-ON* */
1642
1643         if (sm->endpoint_dependent)
1644           {
1645             vec_foreach (ses_index, ses_to_be_removed)
1646             {
1647               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1648               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1649             }
1650           }
1651         else
1652           {
1653             vec_foreach (ses_index, ses_to_be_removed)
1654             {
1655               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1656               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1657             }
1658           }
1659
1660         vec_free (ses_to_be_removed);
1661       }
1662     }
1663
1664 #define _(N, i, n, s) \
1665   vec_free (a->busy_##n##_ports_per_thread);
1666   foreach_nat_protocol
1667 #undef _
1668
1669     if (twice_nat)
1670   {
1671     vec_del1 (sm->twice_nat_addresses, i);
1672     return 0;
1673   }
1674   else vec_del1 (sm->addresses, i);
1675
1676   /* Delete external address from FIB */
1677   pool_foreach (interface, sm->interfaces)
1678     {
1679       if (nat_interface_is_inside (interface) || sm->out2in_dpo)
1680         continue;
1681
1682       snat_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
1683       break;
1684     }
1685
1686   pool_foreach (interface, sm->output_feature_interfaces)
1687    {
1688     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1689       continue;
1690
1691     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1692     break;
1693   }
1694
1695   return 0;
1696 }
1697
1698 static void
1699 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1700 {
1701 #define _(x)                                                                  \
1702   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1703                                 sw_if_index);                                 \
1704   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1705   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1706                                 sw_if_index);                                 \
1707   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1708   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1709                                 sw_if_index);                                 \
1710   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1711   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1712                                 sw_if_index);                                 \
1713   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1714   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1715                                 sw_if_index);                                 \
1716   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1717   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1718                                 sw_if_index);                                 \
1719   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1720   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1721                                 sw_if_index);                                 \
1722   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1723   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1724                                 sw_if_index);                                 \
1725   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1726   foreach_nat_counter;
1727 #undef _
1728   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1729   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1730 }
1731
1732 void
1733 expire_per_vrf_sessions (u32 fib_index)
1734 {
1735   per_vrf_sessions_t *per_vrf_sessions;
1736   snat_main_per_thread_data_t *tsm;
1737   snat_main_t *sm = &snat_main;
1738
1739   /* *INDENT-OFF* */
1740   vec_foreach (tsm, sm->per_thread_data)
1741     {
1742       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1743         {
1744           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1745               (per_vrf_sessions->tx_fib_index == fib_index))
1746             {
1747               per_vrf_sessions->expired = 1;
1748             }
1749         }
1750     }
1751   /* *INDENT-ON* */
1752 }
1753
1754 void
1755 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1756 {
1757   snat_main_t *sm = &snat_main;
1758   nat_fib_t *fib;
1759
1760   // we don't care if it is outside/inside fib
1761   // we just care about their ref_count
1762   // if it reaches 0 sessions should expire
1763   // because the fib isn't valid for NAT anymore
1764
1765   vec_foreach (fib, sm->fibs)
1766   {
1767     if (fib->fib_index == fib_index)
1768       {
1769         if (is_del)
1770           {
1771             fib->ref_count--;
1772             if (!fib->ref_count)
1773               {
1774                 vec_del1 (sm->fibs, fib - sm->fibs);
1775                 expire_per_vrf_sessions (fib_index);
1776               }
1777             return;
1778           }
1779         else
1780           fib->ref_count++;
1781       }
1782   }
1783   if (!is_del)
1784     {
1785       vec_add2 (sm->fibs, fib, 1);
1786       fib->ref_count = 1;
1787       fib->fib_index = fib_index;
1788     }
1789 }
1790
1791 int
1792 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1793 {
1794   snat_main_t *sm = &snat_main;
1795   snat_interface_t *i;
1796   const char *feature_name, *del_feature_name;
1797   snat_address_t *ap;
1798   snat_static_mapping_t *m;
1799   nat_outside_fib_t *outside_fib;
1800   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1801                                                        sw_if_index);
1802
1803   if (!sm->enabled)
1804     {
1805       nat_log_err ("nat44 is disabled");
1806       return VNET_API_ERROR_UNSUPPORTED;
1807     }
1808
1809   if (sm->out2in_dpo && !is_inside)
1810     {
1811       nat_log_err ("error unsupported");
1812       return VNET_API_ERROR_UNSUPPORTED;
1813     }
1814
1815   /* *INDENT-OFF* */
1816   pool_foreach (i, sm->output_feature_interfaces)
1817    {
1818     if (i->sw_if_index == sw_if_index)
1819       {
1820         nat_log_err ("error interface already configured");
1821         return VNET_API_ERROR_VALUE_EXIST;
1822       }
1823   }
1824   /* *INDENT-ON* */
1825
1826   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1827     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1828   else
1829     {
1830       if (sm->num_workers > 1)
1831         feature_name =
1832           is_inside ? "nat44-in2out-worker-handoff" :
1833           "nat44-out2in-worker-handoff";
1834       else if (sm->endpoint_dependent)
1835         {
1836           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1837         }
1838       else
1839         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1840     }
1841
1842   ASSERT (sm->frame_queue_nelts > 0);
1843
1844   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
1845     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1846                                                       sm->frame_queue_nelts);
1847
1848   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1849     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1850                                                       sm->frame_queue_nelts);
1851
1852   if (sm->endpoint_dependent)
1853     update_per_vrf_sessions_vec (fib_index, is_del);
1854
1855   if (!is_inside)
1856     {
1857       /* *INDENT-OFF* */
1858       vec_foreach (outside_fib, sm->outside_fibs)
1859         {
1860           if (outside_fib->fib_index == fib_index)
1861             {
1862               if (is_del)
1863                 {
1864                   outside_fib->refcount--;
1865                   if (!outside_fib->refcount)
1866                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1867                 }
1868               else
1869                 outside_fib->refcount++;
1870               goto feature_set;
1871             }
1872         }
1873       /* *INDENT-ON* */
1874       if (!is_del)
1875         {
1876           vec_add2 (sm->outside_fibs, outside_fib, 1);
1877           outside_fib->refcount = 1;
1878           outside_fib->fib_index = fib_index;
1879         }
1880     }
1881
1882 feature_set:
1883   /* *INDENT-OFF* */
1884   pool_foreach (i, sm->interfaces)
1885    {
1886     if (i->sw_if_index == sw_if_index)
1887       {
1888         if (is_del)
1889           {
1890             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1891               {
1892                 if (is_inside)
1893                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1894                 else
1895                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1896
1897                 if (sm->num_workers > 1)
1898                   {
1899                     del_feature_name = "nat44-handoff-classify";
1900                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1901                                                  "nat44-out2in-worker-handoff";
1902                   }
1903                 else if (sm->endpoint_dependent)
1904                   {
1905                     del_feature_name = "nat44-ed-classify";
1906                     feature_name = !is_inside ?  "nat-pre-in2out" :
1907                                                  "nat-pre-out2in";
1908                   }
1909                 else
1910                   {
1911                     del_feature_name = "nat44-classify";
1912                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1913                   }
1914
1915                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1916                 if (rv)
1917                   return rv;
1918                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1919                                              sw_if_index, 0, 0, 0);
1920                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1921                                              sw_if_index, 1, 0, 0);
1922                 if (!is_inside)
1923                   {
1924                     if (!sm->endpoint_dependent)
1925                       vnet_feature_enable_disable ("ip4-local",
1926                                                    "nat44-hairpinning",
1927                                                    sw_if_index, 1, 0, 0);
1928                   }
1929               }
1930             else
1931               {
1932                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1933                 if (rv)
1934                   return rv;
1935                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1936                                              sw_if_index, 0, 0, 0);
1937                 pool_put (sm->interfaces, i);
1938                 if (is_inside)
1939                   {
1940                     if (!sm->endpoint_dependent)
1941                       vnet_feature_enable_disable ("ip4-local",
1942                                                    "nat44-hairpinning",
1943                                                    sw_if_index, 0, 0, 0);
1944                   }
1945               }
1946           }
1947         else
1948           {
1949             if ((nat_interface_is_inside (i) && is_inside) ||
1950                 (nat_interface_is_outside (i) && !is_inside))
1951               return 0;
1952
1953             if (sm->num_workers > 1)
1954               {
1955                 del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1956                                                 "nat44-out2in-worker-handoff";
1957                 feature_name = "nat44-handoff-classify";
1958               }
1959             else if (sm->endpoint_dependent)
1960               {
1961                 del_feature_name =
1962                   !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1963
1964                 feature_name = "nat44-ed-classify";
1965               }
1966             else
1967               {
1968                 del_feature_name =
1969                   !is_inside ? "nat44-in2out" : "nat44-out2in";
1970                 feature_name = "nat44-classify";
1971               }
1972
1973             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1974             if (rv)
1975               return rv;
1976             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1977                                          sw_if_index, 0, 0, 0);
1978             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1979                                          sw_if_index, 1, 0, 0);
1980             if (!is_inside)
1981               {
1982                 if (!sm->endpoint_dependent)
1983                   vnet_feature_enable_disable (
1984                     "ip4-local", "nat44-hairpinning", sw_if_index, 0, 0, 0);
1985               }
1986             goto set_flags;
1987           }
1988
1989         goto fib;
1990       }
1991   }
1992   /* *INDENT-ON* */
1993
1994   if (is_del)
1995     {
1996       nat_log_err ("error interface couldn't be found");
1997       return VNET_API_ERROR_NO_SUCH_ENTRY;
1998     }
1999
2000   pool_get (sm->interfaces, i);
2001   i->sw_if_index = sw_if_index;
2002   i->flags = 0;
2003   nat_validate_counters (sm, sw_if_index);
2004
2005   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2006                                0);
2007
2008   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2009   if (rv)
2010     return rv;
2011
2012   if (is_inside && !sm->out2in_dpo)
2013     {
2014       if (!sm->endpoint_dependent)
2015         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2016                                      sw_if_index, 1, 0, 0);
2017     }
2018
2019 set_flags:
2020   if (is_inside)
2021     {
2022       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2023       return 0;
2024     }
2025   else
2026     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2027
2028   /* Add/delete external addresses to FIB */
2029 fib:
2030   /* *INDENT-OFF* */
2031   vec_foreach (ap, sm->addresses)
2032     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2033
2034   pool_foreach (m, sm->static_mappings)
2035    {
2036     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2037       continue;
2038
2039     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2040   }
2041   /* *INDENT-ON* */
2042
2043   return 0;
2044 }
2045
2046 int
2047 snat_interface_add_del_output_feature (u32 sw_if_index,
2048                                        u8 is_inside, int is_del)
2049 {
2050   snat_main_t *sm = &snat_main;
2051   snat_interface_t *i;
2052   snat_address_t *ap;
2053   snat_static_mapping_t *m;
2054   nat_outside_fib_t *outside_fib;
2055   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2056                                                        sw_if_index);
2057
2058   if (!sm->enabled)
2059     {
2060       nat_log_err ("nat44 is disabled");
2061       return VNET_API_ERROR_UNSUPPORTED;
2062     }
2063
2064   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2065     {
2066       nat_log_err ("error unsupported");
2067       return VNET_API_ERROR_UNSUPPORTED;
2068     }
2069
2070   /* *INDENT-OFF* */
2071   pool_foreach (i, sm->interfaces)
2072    {
2073     if (i->sw_if_index == sw_if_index)
2074       {
2075         nat_log_err ("error interface already configured");
2076         return VNET_API_ERROR_VALUE_EXIST;
2077       }
2078   }
2079   /* *INDENT-ON* */
2080
2081   if (sm->endpoint_dependent)
2082     update_per_vrf_sessions_vec (fib_index, is_del);
2083
2084   if (!is_inside)
2085     {
2086       /* *INDENT-OFF* */
2087       vec_foreach (outside_fib, sm->outside_fibs)
2088         {
2089           if (outside_fib->fib_index == fib_index)
2090             {
2091               if (is_del)
2092                 {
2093                   outside_fib->refcount--;
2094                   if (!outside_fib->refcount)
2095                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2096                 }
2097               else
2098                 outside_fib->refcount++;
2099               goto feature_set;
2100             }
2101         }
2102       /* *INDENT-ON* */
2103       if (!is_del)
2104         {
2105           vec_add2 (sm->outside_fibs, outside_fib, 1);
2106           outside_fib->refcount = 1;
2107           outside_fib->fib_index = fib_index;
2108         }
2109     }
2110
2111 feature_set:
2112   if (is_inside)
2113     {
2114       if (sm->endpoint_dependent)
2115         {
2116           int rv =
2117             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2118           if (rv)
2119             return rv;
2120           rv =
2121             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2122                                                             !is_del);
2123           if (rv)
2124             return rv;
2125         }
2126       else
2127         {
2128           int rv =
2129             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2130           if (rv)
2131             return rv;
2132           rv =
2133             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2134                                                             !is_del);
2135           if (rv)
2136             return rv;
2137           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2138                                        sw_if_index, !is_del, 0, 0);
2139           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2140                                        sw_if_index, !is_del, 0, 0);
2141         }
2142       goto fq;
2143     }
2144
2145   if (sm->num_workers > 1)
2146     {
2147       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2148       if (rv)
2149         return rv;
2150       rv =
2151         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2152       if (rv)
2153         return rv;
2154       vnet_feature_enable_disable ("ip4-unicast",
2155                                    "nat44-out2in-worker-handoff",
2156                                    sw_if_index, !is_del, 0, 0);
2157       vnet_feature_enable_disable ("ip4-output",
2158                                    "nat44-in2out-output-worker-handoff",
2159                                    sw_if_index, !is_del, 0, 0);
2160     }
2161   else
2162     {
2163       if (sm->endpoint_dependent)
2164         {
2165           int rv =
2166             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2167           if (rv)
2168             return rv;
2169           rv =
2170             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2171                                                             !is_del);
2172           if (rv)
2173             return rv;
2174           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2175                                        sw_if_index, !is_del, 0, 0);
2176           vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2177                                        sw_if_index, !is_del, 0, 0);
2178         }
2179       else
2180         {
2181           int rv =
2182             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2183           if (rv)
2184             return rv;
2185           rv =
2186             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2187                                                             !is_del);
2188           if (rv)
2189             return rv;
2190           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2191                                        sw_if_index, !is_del, 0, 0);
2192           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2193                                        sw_if_index, !is_del, 0, 0);
2194         }
2195     }
2196
2197 fq:
2198   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2199     sm->fq_in2out_output_index =
2200       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2201
2202   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2203     sm->fq_out2in_index =
2204       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2205
2206   /* *INDENT-OFF* */
2207   pool_foreach (i, sm->output_feature_interfaces)
2208    {
2209     if (i->sw_if_index == sw_if_index)
2210       {
2211         if (is_del)
2212           pool_put (sm->output_feature_interfaces, i);
2213         else
2214           return VNET_API_ERROR_VALUE_EXIST;
2215
2216         goto fib;
2217       }
2218   }
2219   /* *INDENT-ON* */
2220
2221   if (is_del)
2222     {
2223       nat_log_err ("error interface couldn't be found");
2224       return VNET_API_ERROR_NO_SUCH_ENTRY;
2225     }
2226
2227   pool_get (sm->output_feature_interfaces, i);
2228   i->sw_if_index = sw_if_index;
2229   i->flags = 0;
2230   nat_validate_counters (sm, sw_if_index);
2231   if (is_inside)
2232     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2233   else
2234     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2235
2236   /* Add/delete external addresses to FIB */
2237 fib:
2238   if (is_inside)
2239     return 0;
2240
2241   /* *INDENT-OFF* */
2242   vec_foreach (ap, sm->addresses)
2243     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2244
2245   pool_foreach (m, sm->static_mappings)
2246    {
2247     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2248       continue;
2249
2250     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2251   }
2252   /* *INDENT-ON* */
2253
2254   return 0;
2255 }
2256
2257 int
2258 snat_set_workers (uword * bitmap)
2259 {
2260   snat_main_t *sm = &snat_main;
2261   int i, j = 0;
2262
2263   if (sm->num_workers < 2)
2264     return VNET_API_ERROR_FEATURE_DISABLED;
2265
2266   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2267     return VNET_API_ERROR_INVALID_WORKER;
2268
2269   vec_free (sm->workers);
2270   /* *INDENT-OFF* */
2271   clib_bitmap_foreach (i, bitmap)
2272     {
2273       vec_add1(sm->workers, i);
2274       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2275       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2276       j++;
2277     }
2278   /* *INDENT-ON* */
2279
2280   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2281
2282   return 0;
2283 }
2284
2285 int
2286 snat_set_frame_queue_nelts (u32 frame_queue_nelts)
2287 {
2288   fail_if_enabled ();
2289   snat_main_t *sm = &snat_main;
2290   sm->frame_queue_nelts = frame_queue_nelts;
2291   return 0;
2292 }
2293
2294 static void
2295 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2296                          u32 sw_if_index, u32 new_fib_index,
2297                          u32 old_fib_index)
2298 {
2299   snat_main_t *sm = &snat_main;
2300   nat_outside_fib_t *outside_fib;
2301   snat_interface_t *i;
2302   u8 is_add = 1;
2303   u8 match = 0;
2304
2305   if (!sm->enabled || (new_fib_index == old_fib_index)
2306       || (!vec_len (sm->outside_fibs)))
2307     {
2308       return;
2309     }
2310
2311   /* *INDENT-OFF* */
2312   pool_foreach (i, sm->interfaces)
2313      {
2314       if (i->sw_if_index == sw_if_index)
2315         {
2316           if (!(nat_interface_is_outside (i)))
2317             return;
2318           match = 1;
2319         }
2320     }
2321
2322   pool_foreach (i, sm->output_feature_interfaces)
2323      {
2324       if (i->sw_if_index == sw_if_index)
2325         {
2326           if (!(nat_interface_is_outside (i)))
2327             return;
2328           match = 1;
2329         }
2330     }
2331   /* *INDENT-ON* */
2332
2333   if (!match)
2334     return;
2335
2336   vec_foreach (outside_fib, sm->outside_fibs)
2337   {
2338     if (outside_fib->fib_index == old_fib_index)
2339       {
2340         outside_fib->refcount--;
2341         if (!outside_fib->refcount)
2342           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2343         break;
2344       }
2345   }
2346
2347   vec_foreach (outside_fib, sm->outside_fibs)
2348   {
2349     if (outside_fib->fib_index == new_fib_index)
2350       {
2351         outside_fib->refcount++;
2352         is_add = 0;
2353         break;
2354       }
2355   }
2356
2357   if (is_add)
2358     {
2359       vec_add2 (sm->outside_fibs, outside_fib, 1);
2360       outside_fib->refcount = 1;
2361       outside_fib->fib_index = new_fib_index;
2362     }
2363 }
2364
2365 static void
2366 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2367                          u32 sw_if_index, u32 new_fib_index,
2368                          u32 old_fib_index);
2369
2370 static void
2371 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2372                                        uword opaque,
2373                                        u32 sw_if_index,
2374                                        ip4_address_t * address,
2375                                        u32 address_length,
2376                                        u32 if_address_index, u32 is_delete);
2377
2378 static void
2379 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2380                                  uword opaque,
2381                                  u32 sw_if_index,
2382                                  ip4_address_t * address,
2383                                  u32 address_length,
2384                                  u32 if_address_index, u32 is_delete);
2385
2386 void
2387 test_key_calc_split ()
2388 {
2389   ip4_address_t l_addr;
2390   l_addr.as_u8[0] = 1;
2391   l_addr.as_u8[1] = 1;
2392   l_addr.as_u8[2] = 1;
2393   l_addr.as_u8[3] = 1;
2394   ip4_address_t r_addr;
2395   r_addr.as_u8[0] = 2;
2396   r_addr.as_u8[1] = 2;
2397   r_addr.as_u8[2] = 2;
2398   r_addr.as_u8[3] = 2;
2399   u16 l_port = 40001;
2400   u16 r_port = 40301;
2401   u8 proto = 9;
2402   u32 fib_index = 9000001;
2403   u32 thread_index = 3000000001;
2404   u32 session_index = 3000000221;
2405   clib_bihash_kv_16_8_t kv;
2406   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2407               thread_index, session_index);
2408   ip4_address_t l_addr2;
2409   ip4_address_t r_addr2;
2410   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2411   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2412   u16 l_port2 = 0;
2413   u16 r_port2 = 0;
2414   u8 proto2 = 0;
2415   u32 fib_index2 = 0;
2416   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2417                &r_port2);
2418   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2419   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2420   ASSERT (l_port == l_port2);
2421   ASSERT (r_port == r_port2);
2422   ASSERT (proto == proto2);
2423   ASSERT (fib_index == fib_index2);
2424   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2425   ASSERT (session_index == ed_value_get_session_index (&kv));
2426
2427   fib_index = 7001;
2428   proto = 5;
2429   nat_protocol_t proto3 = ~0;
2430   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2431   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2432   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2433   ASSERT (l_port == l_port2);
2434   ASSERT (proto == proto3);
2435   ASSERT (fib_index == fib_index2);
2436 }
2437
2438 static clib_error_t *
2439 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2440 {
2441   snat_main_t *sm = &snat_main;
2442   u32 fib_index;
2443
2444   if (sm->endpoint_dependent)
2445     {
2446       // TODO: consider removing all NAT interfaces
2447       if (!is_add)
2448         {
2449           fib_index = ip4_fib_index_from_table_id (table_id);
2450           if (fib_index != ~0)
2451             expire_per_vrf_sessions (fib_index);
2452         }
2453     }
2454   return 0;
2455 }
2456
2457 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2458
2459 void
2460 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2461 {
2462   vlib_node_t *node;
2463
2464   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2465   sm->ei_out2in_node_index = node->index;
2466   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2467   sm->ei_in2out_node_index = node->index;
2468   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2469   sm->ei_in2out_output_node_index = node->index;
2470
2471   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2472   sm->ed_out2in_node_index = node->index;
2473   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2474   sm->ed_in2out_node_index = node->index;
2475   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2476   sm->ed_in2out_output_node_index = node->index;
2477
2478   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2479   sm->error_node_index = node->index;
2480   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2481   sm->pre_in2out_node_index = node->index;
2482   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2483   sm->pre_out2in_node_index = node->index;
2484   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2485   sm->pre_in2out_node_index = node->index;
2486   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2487   sm->pre_out2in_node_index = node->index;
2488   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2489   sm->in2out_fast_node_index = node->index;
2490   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2491   sm->in2out_slowpath_node_index = node->index;
2492   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2493   sm->in2out_slowpath_output_node_index = node->index;
2494   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2495   sm->ed_in2out_slowpath_node_index = node->index;
2496   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2497   sm->out2in_fast_node_index = node->index;
2498   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2499   sm->ed_out2in_slowpath_node_index = node->index;
2500   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2501   sm->hairpinning_node_index = node->index;
2502   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2503   sm->hairpin_dst_node_index = node->index;
2504   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2505   sm->hairpin_src_node_index = node->index;
2506 }
2507
2508 #define nat_init_simple_counter(c, n, sn) \
2509 do                                        \
2510   {                                       \
2511     c.name = n;                           \
2512     c.stat_segment_name = sn;             \
2513     vlib_validate_simple_counter (&c, 0); \
2514     vlib_zero_simple_counter (&c, 0);     \
2515   } while (0);
2516
2517 static clib_error_t *
2518 nat_init (vlib_main_t * vm)
2519 {
2520   snat_main_t *sm = &snat_main;
2521   vlib_thread_main_t *tm = vlib_get_thread_main ();
2522   vlib_thread_registration_t *tr;
2523   ip4_add_del_interface_address_callback_t cbi = { 0 };
2524   ip4_table_bind_callback_t cbt = { 0 };
2525   u32 i, num_threads = 0;
2526   uword *p, *bitmap = 0;
2527
2528   clib_memset (sm, 0, sizeof (*sm));
2529
2530   // required
2531   sm->vnet_main = vnet_get_main ();
2532   // convenience
2533   sm->ip4_main = &ip4_main;
2534   sm->api_main = vlibapi_get_main ();
2535   sm->ip4_lookup_main = &ip4_main.lookup_main;
2536
2537   // frame queue indices used for handoff
2538   sm->fq_out2in_index = ~0;
2539   sm->fq_in2out_index = ~0;
2540   sm->fq_in2out_output_index = ~0;
2541
2542   sm->log_level = SNAT_LOG_ERROR;
2543
2544   nat44_set_node_indexes (sm, vm);
2545   sm->log_class = vlib_log_register_class ("nat", 0);
2546   nat_ipfix_logging_init (vm);
2547
2548   nat_init_simple_counter (sm->total_users, "total-users",
2549                            "/nat44/total-users");
2550   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2551                            "/nat44/total-sessions");
2552   nat_init_simple_counter (sm->user_limit_reached, "user-limit-reached",
2553                            "/nat44/user-limit-reached");
2554
2555 #define _(x)                                            \
2556   sm->counters.fastpath.in2out.x.name = #x;             \
2557   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2558       "/nat44/in2out/fastpath/" #x;                     \
2559   sm->counters.slowpath.in2out.x.name = #x;             \
2560   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2561       "/nat44/in2out/slowpath/" #x;                     \
2562   sm->counters.fastpath.out2in.x.name = #x;             \
2563   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2564       "/nat44/out2in/fastpath/" #x;                     \
2565   sm->counters.slowpath.out2in.x.name = #x;             \
2566   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2567       "/nat44/out2in/slowpath/" #x;                     \
2568   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2569   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2570       "/nat44/ed/in2out/fastpath/" #x;                  \
2571   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2572   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2573       "/nat44/ed/in2out/slowpath/" #x;                  \
2574   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2575   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2576       "/nat44/ed/out2in/fastpath/" #x;                  \
2577   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2578   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2579       "/nat44/ed/out2in/slowpath/" #x;
2580   foreach_nat_counter;
2581 #undef _
2582   sm->counters.hairpinning.name = "hairpinning";
2583   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2584
2585   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2586   if (p)
2587     {
2588       tr = (vlib_thread_registration_t *) p[0];
2589       if (tr)
2590         {
2591           sm->num_workers = tr->count;
2592           sm->first_worker_index = tr->first_index;
2593         }
2594     }
2595   num_threads = tm->n_vlib_mains - 1;
2596   sm->port_per_thread = 0xffff - 1024;
2597   vec_validate (sm->per_thread_data, num_threads);
2598
2599   /* Use all available workers by default */
2600   if (sm->num_workers > 1)
2601     {
2602
2603       for (i = 0; i < sm->num_workers; i++)
2604         bitmap = clib_bitmap_set (bitmap, i, 1);
2605       snat_set_workers (bitmap);
2606       clib_bitmap_free (bitmap);
2607     }
2608   else
2609     sm->per_thread_data[0].snat_thread_index = 0;
2610
2611   /* callbacks to call when interface address changes. */
2612   cbi.function = snat_ip4_add_del_interface_address_cb;
2613   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2614   cbi.function = nat_ip4_add_del_addr_only_sm_cb;
2615   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2616
2617   /* callbacks to call when interface to table biding changes */
2618   cbt.function = snat_update_outside_fib;
2619   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2620
2621   sm->fib_src_low =
2622     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2623                          FIB_SOURCE_BH_SIMPLE);
2624   sm->fib_src_hi =
2625     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2626                          FIB_SOURCE_BH_SIMPLE);
2627
2628   /* used only by out2in-dpo feature */
2629   nat_dpo_module_init ();
2630
2631   nat_affinity_init (vm);
2632   nat_ha_init (vm, sm->num_workers, num_threads);
2633
2634   test_key_calc_split ();
2635   return nat44_api_hookup (vm);
2636 }
2637
2638 VLIB_INIT_FUNCTION (nat_init);
2639
2640 static int
2641 nat44_ed_plugin_enable (nat44_config_t c)
2642 {
2643   snat_main_t *sm = &snat_main;
2644
2645   if (c.static_mapping_only && !c.connection_tracking)
2646     {
2647       nat_log_err ("unsupported combination of configuration");
2648       return 1;
2649     }
2650
2651   // nat44 feature configuration
2652   sm->endpoint_dependent = c.endpoint_dependent;
2653   sm->static_mapping_only = c.static_mapping_only;
2654   sm->static_mapping_connection_tracking = c.connection_tracking;
2655
2656   // EI only feature (could break ED)
2657   sm->out2in_dpo = c.out2in_dpo;
2658
2659   sm->forwarding_enabled = 0;
2660   sm->mss_clamping = 0;
2661   sm->pat = (!c.static_mapping_only ||
2662              (c.static_mapping_only && c.connection_tracking));
2663
2664   if (!c.users)
2665     c.users = 1024;
2666
2667   // EI only feature (could break ED)
2668   sm->max_users_per_thread = c.users;
2669   sm->user_buckets = nat_calc_bihash_buckets (c.users);
2670
2671   if (!c.sessions)
2672     c.sessions = 10 * 1024;
2673
2674   sm->max_translations_per_thread = c.sessions;
2675   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2676
2677   // ED only feature
2678   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2679
2680   // EI only feature (could break ED)
2681   sm->max_translations_per_user
2682     = c.user_sessions ? c.user_sessions : sm->max_translations_per_thread;
2683
2684   sm->inside_vrf_id = c.inside_vrf;
2685   sm->inside_fib_index =
2686     fib_table_find_or_create_and_lock
2687     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2688
2689   sm->outside_vrf_id = c.outside_vrf;
2690   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2691     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2692
2693   sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
2694   sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2695
2696   sm->in2out_node_index = sm->ed_in2out_node_index;
2697   sm->out2in_node_index = sm->ed_out2in_node_index;
2698
2699   sm->in2out_output_node_index = sm->ed_in2out_output_node_index;
2700
2701   if (sm->pat)
2702     {
2703       sm->icmp_match_in2out_cb = NULL;
2704       sm->icmp_match_out2in_cb = NULL;
2705     }
2706   else
2707     {
2708       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2709       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2710     }
2711
2712   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets,
2713                     sm->user_buckets);
2714
2715   nat_affinity_enable ();
2716
2717   nat_reset_timeouts (&sm->timeouts);
2718
2719   // TODO: function for reset counters
2720   vlib_zero_simple_counter (&sm->total_users, 0);
2721   vlib_zero_simple_counter (&sm->total_sessions, 0);
2722   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2723
2724   if (!sm->frame_queue_nelts)
2725     sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2726
2727   sm->enabled = 1;
2728   sm->rconfig = c;
2729
2730   return 0;
2731 }
2732
2733 int
2734 nat44_plugin_enable (nat44_config_t c)
2735 {
2736   fail_if_enabled ();
2737
2738   // c.static_mapping_only + c.connection_tracking
2739   //  - supported in NAT EI & NAT ED
2740   // c.out2in_dpo, c.static_mapping_only
2741   //  - supported in NAT EI
2742
2743   if (c.endpoint_dependent)
2744     {
2745       if (c.out2in_dpo || c.users || c.user_sessions)
2746         {
2747           nat_log_err ("unsupported combination of configuration");
2748           return 1;
2749         }
2750       return nat44_ed_plugin_enable (c);
2751     }
2752
2753   // separation:
2754   // for now just copy variables
2755
2756   nat44_ei_config_t ei_c = {
2757     .inside_vrf = c.inside_vrf,
2758     .outside_vrf = c.outside_vrf,
2759     .users = c.users,
2760     .sessions = c.sessions,
2761     .user_sessions = c.user_sessions,
2762     .out2in_dpo = c.out2in_dpo,
2763     .static_mapping_only = c.static_mapping_only,
2764     .connection_tracking = c.connection_tracking,
2765   };
2766
2767   return nat44_ei_plugin_enable (ei_c);
2768 }
2769
2770 void
2771 nat44_addresses_free (snat_address_t ** addresses)
2772 {
2773   snat_address_t *ap;
2774   /* *INDENT-OFF* */
2775   vec_foreach (ap, *addresses)
2776     {
2777     #define _(N, i, n, s) \
2778       vec_free (ap->busy_##n##_ports_per_thread);
2779       foreach_nat_protocol
2780     #undef _
2781     }
2782   /* *INDENT-ON* */
2783   vec_free (*addresses);
2784   *addresses = 0;
2785 }
2786
2787 static int
2788 nat44_ed_plugin_disable ()
2789 {
2790   snat_main_t *sm = &snat_main;
2791   snat_interface_t *i, *vec;
2792   int error = 0;
2793
2794   // first unregister all nodes from interfaces
2795   vec = vec_dup (sm->interfaces);
2796   /* *INDENT-OFF* */
2797   vec_foreach (i, vec)
2798     {
2799       if (nat_interface_is_inside(i))
2800         error = snat_interface_add_del (i->sw_if_index, 1, 1);
2801       if (nat_interface_is_outside(i))
2802         error = snat_interface_add_del (i->sw_if_index, 0, 1);
2803
2804       if (error)
2805         {
2806           nat_log_err ("error occurred while removing interface %u",
2807                        i->sw_if_index);
2808         }
2809     }
2810   /* *INDENT-ON* */
2811   vec_free (vec);
2812   sm->interfaces = 0;
2813
2814   vec = vec_dup (sm->output_feature_interfaces);
2815   /* *INDENT-OFF* */
2816   vec_foreach (i, vec)
2817     {
2818       if (nat_interface_is_inside(i))
2819         error = snat_interface_add_del_output_feature (i->sw_if_index, 1, 1);
2820       if (nat_interface_is_outside(i))
2821         error = snat_interface_add_del_output_feature (i->sw_if_index, 0, 1);
2822
2823       if (error)
2824         {
2825           nat_log_err ("error occurred while removing interface %u",
2826                        i->sw_if_index);
2827         }
2828     }
2829   /* *INDENT-ON* */
2830   vec_free (vec);
2831   sm->output_feature_interfaces = 0;
2832
2833   vec_free (sm->max_translations_per_fib);
2834
2835   nat44_ed_db_free ();
2836
2837   nat44_addresses_free (&sm->addresses);
2838   nat44_addresses_free (&sm->twice_nat_addresses);
2839
2840
2841   vec_free (sm->to_resolve);
2842   vec_free (sm->auto_add_sw_if_indices);
2843   vec_free (sm->auto_add_sw_if_indices_twice_nat);
2844
2845   sm->to_resolve = 0;
2846   sm->auto_add_sw_if_indices = 0;
2847   sm->auto_add_sw_if_indices_twice_nat = 0;
2848
2849   sm->forwarding_enabled = 0;
2850
2851   sm->enabled = 0;
2852   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2853
2854   return 0;
2855 }
2856
2857 int
2858 nat44_plugin_disable ()
2859 {
2860   snat_main_t *sm = &snat_main;
2861
2862   fail_if_disabled ();
2863
2864   if (sm->endpoint_dependent)
2865     return nat44_ed_plugin_disable ();
2866   return nat44_ei_plugin_disable ();
2867 }
2868
2869 void
2870 snat_free_outside_address_and_port (snat_address_t *addresses,
2871                                     u32 thread_index, ip4_address_t *addr,
2872                                     u16 port, nat_protocol_t protocol)
2873 {
2874   snat_address_t *a;
2875   u32 address_index;
2876   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2877
2878   for (address_index = 0; address_index < vec_len (addresses);
2879        address_index++)
2880     {
2881       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2882         break;
2883     }
2884
2885   ASSERT (address_index < vec_len (addresses));
2886
2887   a = addresses + address_index;
2888
2889   switch (protocol)
2890     {
2891 #define _(N, i, n, s) \
2892     case NAT_PROTOCOL_##N: \
2893       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2894       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2895       a->busy_##n##_ports--; \
2896       a->busy_##n##_ports_per_thread[thread_index]--; \
2897       break;
2898       foreach_nat_protocol
2899 #undef _
2900     default:
2901       nat_elog_info ("unknown protocol");
2902       return;
2903     }
2904 }
2905
2906 int
2907 nat_set_outside_address_and_port (snat_address_t *addresses, u32 thread_index,
2908                                   ip4_address_t addr, u16 port,
2909                                   nat_protocol_t protocol)
2910 {
2911   snat_address_t *a = 0;
2912   u32 address_index;
2913   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2914
2915   for (address_index = 0; address_index < vec_len (addresses);
2916        address_index++)
2917     {
2918       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2919         continue;
2920
2921       a = addresses + address_index;
2922       switch (protocol)
2923         {
2924 #define _(N, j, n, s) \
2925         case NAT_PROTOCOL_##N: \
2926           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2927             return VNET_API_ERROR_INSTANCE_IN_USE; \
2928           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2929           a->busy_##n##_ports_per_thread[thread_index]++; \
2930           a->busy_##n##_ports++; \
2931           return 0;
2932           foreach_nat_protocol
2933 #undef _
2934         default:
2935           nat_elog_info ("unknown protocol");
2936           return 1;
2937         }
2938     }
2939
2940   return VNET_API_ERROR_NO_SUCH_ENTRY;
2941 }
2942
2943 int
2944 snat_static_mapping_match (snat_main_t * sm,
2945                            ip4_address_t match_addr,
2946                            u16 match_port,
2947                            u32 match_fib_index,
2948                            nat_protocol_t match_protocol,
2949                            ip4_address_t * mapping_addr,
2950                            u16 * mapping_port,
2951                            u32 * mapping_fib_index,
2952                            u8 by_external,
2953                            u8 * is_addr_only,
2954                            twice_nat_type_t * twice_nat,
2955                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2956                            u8 * is_identity_nat, snat_static_mapping_t ** out)
2957 {
2958   clib_bihash_kv_8_8_t kv, value;
2959   clib_bihash_8_8_t *mapping_hash;
2960   snat_static_mapping_t *m;
2961   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2962   nat44_lb_addr_port_t *local;
2963   u8 backend_index;
2964
2965   if (!by_external)
2966     {
2967       mapping_hash = &sm->static_mapping_by_local;
2968       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2969                   match_protocol);
2970       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2971         {
2972           /* Try address only mapping */
2973           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2974           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2975             return 1;
2976         }
2977     }
2978   else
2979     {
2980       mapping_hash = &sm->static_mapping_by_external;
2981       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2982       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2983         {
2984           /* Try address only mapping */
2985           init_nat_k (&kv, match_addr, 0, 0, 0);
2986           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2987             return 1;
2988         }
2989     }
2990
2991   m = pool_elt_at_index (sm->static_mappings, value.value);
2992
2993   if (by_external)
2994     {
2995       if (is_lb_static_mapping (m))
2996         {
2997           if (PREDICT_FALSE (lb != 0))
2998             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2999           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
3000                                                           match_addr,
3001                                                           match_protocol,
3002                                                           match_port,
3003                                                           &backend_index))
3004             {
3005               local = pool_elt_at_index (m->locals, backend_index);
3006               *mapping_addr = local->addr;
3007               *mapping_port = local->port;
3008               *mapping_fib_index = local->fib_index;
3009               goto end;
3010             }
3011           // pick locals matching this worker
3012           if (PREDICT_FALSE (sm->num_workers > 1))
3013             {
3014               u32 thread_index = vlib_get_thread_index ();
3015               /* *INDENT-OFF* */
3016               pool_foreach_index (i, m->locals)
3017                {
3018                 local = pool_elt_at_index (m->locals, i);
3019
3020                 ip4_header_t ip = {
3021                   .src_address = local->addr,
3022                 };
3023
3024                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
3025                     thread_index)
3026                   {
3027                     vec_add1 (tmp, i);
3028                   }
3029               }
3030               /* *INDENT-ON* */
3031               ASSERT (vec_len (tmp) != 0);
3032             }
3033           else
3034             {
3035               /* *INDENT-OFF* */
3036               pool_foreach_index (i, m->locals)
3037                {
3038                 vec_add1 (tmp, i);
3039               }
3040               /* *INDENT-ON* */
3041             }
3042           hi = vec_len (tmp) - 1;
3043           local = pool_elt_at_index (m->locals, tmp[hi]);
3044           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
3045           while (lo < hi)
3046             {
3047               mid = ((hi - lo) >> 1) + lo;
3048               local = pool_elt_at_index (m->locals, tmp[mid]);
3049               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
3050             }
3051           local = pool_elt_at_index (m->locals, tmp[lo]);
3052           if (!(local->prefix >= rand))
3053             return 1;
3054           *mapping_addr = local->addr;
3055           *mapping_port = local->port;
3056           *mapping_fib_index = local->fib_index;
3057           if (m->affinity)
3058             {
3059               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
3060                                                 match_protocol, match_port,
3061                                                 tmp[lo], m->affinity,
3062                                                 m->affinity_per_service_list_head_index))
3063                 nat_elog_info ("create affinity record failed");
3064             }
3065           vec_free (tmp);
3066         }
3067       else
3068         {
3069           if (PREDICT_FALSE (lb != 0))
3070             *lb = NO_LB_NAT;
3071           *mapping_fib_index = m->fib_index;
3072           *mapping_addr = m->local_addr;
3073           /* Address only mapping doesn't change port */
3074           *mapping_port = is_addr_only_static_mapping (m) ? match_port
3075             : m->local_port;
3076         }
3077     }
3078   else
3079     {
3080       *mapping_addr = m->external_addr;
3081       /* Address only mapping doesn't change port */
3082       *mapping_port = is_addr_only_static_mapping (m) ? match_port
3083         : m->external_port;
3084       *mapping_fib_index = sm->outside_fib_index;
3085     }
3086
3087 end:
3088   if (PREDICT_FALSE (is_addr_only != 0))
3089     *is_addr_only = is_addr_only_static_mapping (m);
3090
3091   if (PREDICT_FALSE (twice_nat != 0))
3092     *twice_nat = m->twice_nat;
3093
3094   if (PREDICT_FALSE (is_identity_nat != 0))
3095     *is_identity_nat = is_identity_static_mapping (m);
3096
3097   if (out != 0)
3098     *out = m;
3099
3100   return 0;
3101 }
3102
3103 void
3104 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3105 {
3106   snat_main_t *sm = &snat_main;
3107   dpo_id_t dpo_v4 = DPO_INVALID;
3108   fib_prefix_t pfx = {
3109     .fp_proto = FIB_PROTOCOL_IP4,
3110     .fp_len = 32,
3111     .fp_addr.ip4.as_u32 = addr.as_u32,
3112   };
3113
3114   if (is_add)
3115     {
3116       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3117       fib_table_entry_special_dpo_add (0, &pfx, sm->fib_src_hi,
3118                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3119       dpo_reset (&dpo_v4);
3120     }
3121   else
3122     {
3123       fib_table_entry_special_remove (0, &pfx, sm->fib_src_hi);
3124     }
3125 }
3126
3127 static u32
3128 nat44_ed_get_worker_in2out_cb (ip4_header_t *ip, u32 rx_fib_index,
3129                                u8 is_output)
3130 {
3131   snat_main_t *sm = &snat_main;
3132   u32 next_worker_index = sm->first_worker_index;
3133   u32 hash;
3134
3135   clib_bihash_kv_16_8_t kv16, value16;
3136   snat_main_per_thread_data_t *tsm;
3137   udp_header_t *udp;
3138
3139   if (PREDICT_FALSE (is_output))
3140     {
3141       u32 fib_index = sm->outside_fib_index;
3142       nat_outside_fib_t *outside_fib;
3143       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3144       fib_prefix_t pfx = {
3145         .fp_proto = FIB_PROTOCOL_IP4,
3146         .fp_len = 32,
3147         .fp_addr = {
3148                     .ip4.as_u32 = ip->dst_address.as_u32,
3149                     }
3150         ,
3151       };
3152
3153       udp = ip4_next_header (ip);
3154
3155       switch (vec_len (sm->outside_fibs))
3156         {
3157         case 0:
3158           fib_index = sm->outside_fib_index;
3159           break;
3160         case 1:
3161           fib_index = sm->outside_fibs[0].fib_index;
3162           break;
3163         default:
3164             /* *INDENT-OFF* */
3165             vec_foreach (outside_fib, sm->outside_fibs)
3166               {
3167                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3168                 if (FIB_NODE_INDEX_INVALID != fei)
3169                   {
3170                     if (fib_entry_get_resolving_interface (fei) != ~0)
3171                       {
3172                         fib_index = outside_fib->fib_index;
3173                         break;
3174                       }
3175                   }
3176               }
3177             /* *INDENT-ON* */
3178           break;
3179         }
3180
3181       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3182                  udp->dst_port, fib_index, ip->protocol);
3183
3184       if (PREDICT_TRUE (
3185             !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3186         {
3187           tsm =
3188             vec_elt_at_index (sm->per_thread_data,
3189                               ed_value_get_thread_index (&value16));
3190           next_worker_index += tsm->thread_index;
3191
3192           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3193                                   next_worker_index, fib_index,
3194                                   clib_net_to_host_u32 (ip->
3195                                                         src_address.as_u32),
3196                                   clib_net_to_host_u32 (ip->
3197                                                         dst_address.as_u32));
3198
3199           return next_worker_index;
3200         }
3201     }
3202
3203   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3204     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3205
3206   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3207     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3208   else
3209     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3210
3211   if (PREDICT_TRUE (!is_output))
3212     {
3213       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3214                               next_worker_index, rx_fib_index,
3215                               clib_net_to_host_u32 (ip->src_address.as_u32),
3216                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3217     }
3218   else
3219     {
3220       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3221                               next_worker_index, rx_fib_index,
3222                               clib_net_to_host_u32 (ip->src_address.as_u32),
3223                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3224     }
3225
3226   return next_worker_index;
3227 }
3228
3229 static u32
3230 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3231                                u32 rx_fib_index, u8 is_output)
3232 {
3233   snat_main_t *sm = &snat_main;
3234   clib_bihash_kv_8_8_t kv, value;
3235   clib_bihash_kv_16_8_t kv16, value16;
3236   snat_main_per_thread_data_t *tsm;
3237
3238   u32 proto, next_worker_index = 0;
3239   udp_header_t *udp;
3240   u16 port;
3241   snat_static_mapping_t *m;
3242   u32 hash;
3243
3244   proto = ip_proto_to_nat_proto (ip->protocol);
3245
3246   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3247     {
3248       udp = ip4_next_header (ip);
3249
3250       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3251                  udp->src_port, rx_fib_index, ip->protocol);
3252
3253       if (PREDICT_TRUE (
3254             !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3255         {
3256           tsm =
3257             vec_elt_at_index (sm->per_thread_data,
3258                               ed_value_get_thread_index (&value16));
3259           vnet_buffer2 (b)->nat.cached_session_index =
3260             ed_value_get_session_index (&value16);
3261           next_worker_index = sm->first_worker_index + tsm->thread_index;
3262           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3263                                   next_worker_index, rx_fib_index,
3264                                   clib_net_to_host_u32 (ip->
3265                                                         src_address.as_u32),
3266                                   clib_net_to_host_u32 (ip->
3267                                                         dst_address.as_u32));
3268           return next_worker_index;
3269         }
3270     }
3271   else if (proto == NAT_PROTOCOL_ICMP)
3272     {
3273       ip4_address_t lookup_saddr, lookup_daddr;
3274       u16 lookup_sport, lookup_dport;
3275       u8 lookup_protocol;
3276       if (!nat_get_icmp_session_lookup_values (
3277             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3278             &lookup_protocol))
3279         {
3280           init_ed_k (&kv16, lookup_saddr, lookup_sport, lookup_daddr,
3281                      lookup_dport, rx_fib_index, lookup_protocol);
3282           if (PREDICT_TRUE (
3283                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3284             {
3285               tsm =
3286                 vec_elt_at_index (sm->per_thread_data,
3287                                   ed_value_get_thread_index (&value16));
3288               next_worker_index = sm->first_worker_index + tsm->thread_index;
3289               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3290                                       next_worker_index, rx_fib_index,
3291                                       clib_net_to_host_u32 (ip->
3292                                                             src_address.as_u32),
3293                                       clib_net_to_host_u32 (ip->
3294                                                             dst_address.as_u32));
3295               return next_worker_index;
3296             }
3297         }
3298     }
3299
3300   /* first try static mappings without port */
3301   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3302     {
3303       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3304       if (!clib_bihash_search_8_8
3305           (&sm->static_mapping_by_external, &kv, &value))
3306         {
3307           m = pool_elt_at_index (sm->static_mappings, value.value);
3308           next_worker_index = m->workers[0];
3309           goto done;
3310         }
3311     }
3312
3313   /* unknown protocol */
3314   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3315     {
3316       /* use current thread */
3317       next_worker_index = vlib_get_thread_index ();
3318       goto done;
3319     }
3320
3321   udp = ip4_next_header (ip);
3322   port = udp->dst_port;
3323
3324   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3325     {
3326       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3327       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3328       if (!icmp_type_is_error_message
3329           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3330         port = vnet_buffer (b)->ip.reass.l4_src_port;
3331       else
3332         {
3333           /* if error message, then it's not fragmented and we can access it */
3334           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3335           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3336           void *l4_header = ip4_next_header (inner_ip);
3337           switch (proto)
3338             {
3339             case NAT_PROTOCOL_ICMP:
3340               icmp = (icmp46_header_t *) l4_header;
3341               echo = (icmp_echo_header_t *) (icmp + 1);
3342               port = echo->identifier;
3343               break;
3344             case NAT_PROTOCOL_UDP:
3345             case NAT_PROTOCOL_TCP:
3346               port = ((tcp_udp_header_t *) l4_header)->src_port;
3347               break;
3348             default:
3349               next_worker_index = vlib_get_thread_index ();
3350               goto done;
3351             }
3352         }
3353     }
3354
3355   /* try static mappings with port */
3356   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3357     {
3358       init_nat_k (&kv, ip->dst_address, port, 0, proto);
3359       if (!clib_bihash_search_8_8
3360           (&sm->static_mapping_by_external, &kv, &value))
3361         {
3362           m = pool_elt_at_index (sm->static_mappings, value.value);
3363           if (!is_lb_static_mapping (m))
3364             {
3365               next_worker_index = m->workers[0];
3366               goto done;
3367             }
3368
3369           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3370             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3371
3372           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3373             next_worker_index =
3374               m->workers[hash & (_vec_len (m->workers) - 1)];
3375           else
3376             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3377           goto done;
3378         }
3379     }
3380
3381   /* worker by outside port */
3382   next_worker_index = sm->first_worker_index;
3383   next_worker_index +=
3384     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3385
3386 done:
3387   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3388                           clib_net_to_host_u32 (ip->src_address.as_u32),
3389                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3390   return next_worker_index;
3391 }
3392
3393 u32
3394 nat_calc_bihash_buckets (u32 n_elts)
3395 {
3396   n_elts = n_elts / 2.5;
3397   u64 lower_pow2 = 1;
3398   while (lower_pow2 * 2 < n_elts)
3399     {
3400       lower_pow2 = 2 * lower_pow2;
3401     }
3402   u64 upper_pow2 = 2 * lower_pow2;
3403   if ((upper_pow2 - n_elts) < (n_elts - lower_pow2))
3404     {
3405       if (upper_pow2 <= UINT32_MAX)
3406         {
3407           return upper_pow2;
3408         }
3409     }
3410   return lower_pow2;
3411 }
3412
3413 u32
3414 nat44_get_max_session_limit ()
3415 {
3416   snat_main_t *sm = &snat_main;
3417   u32 max_limit = 0, len = 0;
3418
3419   for (; len < vec_len (sm->max_translations_per_fib); len++)
3420     {
3421       if (max_limit < sm->max_translations_per_fib[len])
3422         max_limit = sm->max_translations_per_fib[len];
3423     }
3424   return max_limit;
3425 }
3426
3427 int
3428 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3429 {
3430   snat_main_t *sm = &snat_main;
3431   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3432   u32 len = vec_len (sm->max_translations_per_fib);
3433
3434   if (len <= fib_index)
3435     {
3436       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3437
3438       for (; len < vec_len (sm->max_translations_per_fib); len++)
3439         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3440     }
3441
3442   sm->max_translations_per_fib[fib_index] = session_limit;
3443   return 0;
3444 }
3445
3446 int
3447 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3448 {
3449   snat_main_t *sm = &snat_main;
3450
3451   if (nat44_set_session_limit (session_limit, vrf_id))
3452     return 1;
3453   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3454
3455   sm->translation_buckets =
3456     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3457
3458   nat44_sessions_clear ();
3459   return 0;
3460 }
3461
3462 static void
3463 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
3464                          u32 translation_buckets, u32 user_buckets)
3465 {
3466   dlist_elt_t *head;
3467
3468   pool_alloc (tsm->sessions, translations);
3469   pool_alloc (tsm->lru_pool, translations);
3470
3471   pool_get (tsm->lru_pool, head);
3472   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3473   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3474
3475   pool_get (tsm->lru_pool, head);
3476   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3477   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3478
3479   pool_get (tsm->lru_pool, head);
3480   tsm->udp_lru_head_index = head - tsm->lru_pool;
3481   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3482
3483   pool_get (tsm->lru_pool, head);
3484   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3485   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3486
3487   pool_get (tsm->lru_pool, head);
3488   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3489   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3490
3491   // TODO: ED nat is not using these
3492   // before removal large refactor required
3493   pool_alloc (tsm->list_pool, translations);
3494   clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets, 0);
3495   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3496 }
3497
3498 static void
3499 reinit_ed_flow_hash ()
3500 {
3501   snat_main_t *sm = &snat_main;
3502   // we expect 2 flows per session, so multiply translation_buckets by 2
3503   clib_bihash_init_16_8 (
3504     &sm->flow_hash, "ed-flow-hash",
3505     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3506   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3507 }
3508
3509 static void
3510 nat44_ed_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
3511 {
3512   snat_main_t *sm = &snat_main;
3513   snat_main_per_thread_data_t *tsm;
3514   u32 static_mapping_buckets = 1024;
3515   u32 static_mapping_memory_size = 64 << 20;
3516
3517   reinit_ed_flow_hash ();
3518
3519   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3520                         "static_mapping_by_local", static_mapping_buckets,
3521                         static_mapping_memory_size);
3522   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3523                                      format_static_mapping_kvp);
3524
3525   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3526                         "static_mapping_by_external", static_mapping_buckets,
3527                         static_mapping_memory_size);
3528   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3529                                      format_static_mapping_kvp);
3530
3531   if (sm->pat)
3532     {
3533       vec_foreach (tsm, sm->per_thread_data)
3534         {
3535           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3536                                    sm->translation_buckets, sm->user_buckets);
3537         }
3538     }
3539 }
3540
3541 static void
3542 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3543 {
3544   pool_free (tsm->sessions);
3545   pool_free (tsm->lru_pool);
3546
3547   vec_free (tsm->per_vrf_sessions_vec);
3548
3549   // TODO: resolve static mappings (put only to !ED)
3550   pool_free (tsm->list_pool);
3551   pool_free (tsm->users);
3552   clib_bihash_free_8_8 (&tsm->user_hash);
3553 }
3554
3555 static void
3556 nat44_ed_db_free ()
3557 {
3558   snat_main_t *sm = &snat_main;
3559   snat_main_per_thread_data_t *tsm;
3560
3561   pool_free (sm->static_mappings);
3562   clib_bihash_free_16_8 (&sm->flow_hash);
3563   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
3564   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
3565
3566   if (sm->pat)
3567     {
3568       vec_foreach (tsm, sm->per_thread_data)
3569         {
3570           nat44_ed_worker_db_free (tsm);
3571         }
3572     }
3573 }
3574
3575 void
3576 nat44_ed_sessions_clear ()
3577 {
3578   snat_main_t *sm = &snat_main;
3579   snat_main_per_thread_data_t *tsm;
3580
3581   reinit_ed_flow_hash ();
3582
3583   if (sm->pat)
3584     {
3585       vec_foreach (tsm, sm->per_thread_data)
3586         {
3587
3588           nat44_ed_worker_db_free (tsm);
3589           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3590                                    sm->translation_buckets, sm->user_buckets);
3591         }
3592     }
3593
3594   // TODO: function for reset counters
3595   vlib_zero_simple_counter (&sm->total_users, 0);
3596   vlib_zero_simple_counter (&sm->total_sessions, 0);
3597   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
3598 }
3599
3600 void
3601 nat44_sessions_clear ()
3602 {
3603   snat_main_t *sm = &snat_main;
3604
3605   if (sm->endpoint_dependent)
3606     nat44_ed_sessions_clear ();
3607   else
3608     nat44_ei_sessions_clear ();
3609 }
3610
3611 static void
3612 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3613                                  uword opaque,
3614                                  u32 sw_if_index,
3615                                  ip4_address_t * address,
3616                                  u32 address_length,
3617                                  u32 if_address_index, u32 is_delete)
3618 {
3619   snat_main_t *sm = &snat_main;
3620   snat_static_map_resolve_t *rp;
3621   snat_static_mapping_t *m;
3622   clib_bihash_kv_8_8_t kv, value;
3623   int i, rv;
3624   ip4_address_t l_addr;
3625
3626   if (!sm->enabled)
3627     return;
3628
3629   for (i = 0; i < vec_len (sm->to_resolve); i++)
3630     {
3631       rp = sm->to_resolve + i;
3632       if (rp->addr_only == 0)
3633         continue;
3634       if (rp->sw_if_index == sw_if_index)
3635         goto match;
3636     }
3637
3638   return;
3639
3640 match:
3641   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
3642               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
3643   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3644     m = 0;
3645   else
3646     m = pool_elt_at_index (sm->static_mappings, value.value);
3647
3648   if (!is_delete)
3649     {
3650       /* Don't trip over lease renewal, static config */
3651       if (m)
3652         return;
3653     }
3654   else
3655     {
3656       if (!m)
3657         return;
3658     }
3659
3660   /* Indetity mapping? */
3661   if (rp->l_addr.as_u32 == 0)
3662     l_addr.as_u32 = address[0].as_u32;
3663   else
3664     l_addr.as_u32 = rp->l_addr.as_u32;
3665   /* Add the static mapping */
3666   rv = snat_add_static_mapping (l_addr,
3667                                 address[0],
3668                                 rp->l_port,
3669                                 rp->e_port,
3670                                 rp->vrf_id,
3671                                 rp->addr_only, ~0 /* sw_if_index */ ,
3672                                 rp->proto, !is_delete, rp->twice_nat,
3673                                 rp->out2in_only, rp->tag, rp->identity_nat,
3674                                 rp->pool_addr, rp->exact);
3675   if (rv)
3676     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
3677 }
3678
3679 static void
3680 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3681                                        uword opaque,
3682                                        u32 sw_if_index,
3683                                        ip4_address_t * address,
3684                                        u32 address_length,
3685                                        u32 if_address_index, u32 is_delete)
3686 {
3687   snat_main_t *sm = &snat_main;
3688   snat_static_map_resolve_t *rp;
3689   ip4_address_t l_addr;
3690   int i, j;
3691   int rv;
3692   u8 twice_nat = 0;
3693   snat_address_t *addresses = sm->addresses;
3694
3695   if (!sm->enabled)
3696     return;
3697
3698   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3699     {
3700       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3701         goto match;
3702     }
3703
3704   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3705     {
3706       twice_nat = 1;
3707       addresses = sm->twice_nat_addresses;
3708       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3709         goto match;
3710     }
3711
3712   return;
3713
3714 match:
3715   if (!is_delete)
3716     {
3717       /* Don't trip over lease renewal, static config */
3718       for (j = 0; j < vec_len (addresses); j++)
3719         if (addresses[j].addr.as_u32 == address->as_u32)
3720           return;
3721
3722       (void) snat_add_address (sm, address, ~0, twice_nat);
3723       /* Scan static map resolution vector */
3724       for (j = 0; j < vec_len (sm->to_resolve); j++)
3725         {
3726           rp = sm->to_resolve + j;
3727           if (rp->addr_only)
3728             continue;
3729           /* On this interface? */
3730           if (rp->sw_if_index == sw_if_index)
3731             {
3732               /* Indetity mapping? */
3733               if (rp->l_addr.as_u32 == 0)
3734                 l_addr.as_u32 = address[0].as_u32;
3735               else
3736                 l_addr.as_u32 = rp->l_addr.as_u32;
3737               /* Add the static mapping */
3738               rv = snat_add_static_mapping (
3739                 l_addr, address[0], rp->l_port, rp->e_port, rp->vrf_id,
3740                 rp->addr_only, ~0 /* sw_if_index */, rp->proto, 1,
3741                 rp->twice_nat, rp->out2in_only, rp->tag, rp->identity_nat,
3742                 rp->pool_addr, rp->exact);
3743               if (rv)
3744                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
3745                                     "i4", rv);
3746             }
3747         }
3748       return;
3749     }
3750   else
3751     {
3752       (void) snat_del_address (sm, address[0], 1, twice_nat);
3753       return;
3754     }
3755 }
3756
3757 int
3758 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3759                             u8 twice_nat)
3760 {
3761   ip4_main_t *ip4_main = sm->ip4_main;
3762   ip4_address_t *first_int_addr;
3763   snat_static_map_resolve_t *rp;
3764   u32 *indices_to_delete = 0;
3765   int i, j;
3766   u32 *auto_add_sw_if_indices =
3767     twice_nat ? sm->
3768     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3769
3770   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3771     );
3772
3773   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3774     {
3775       if (auto_add_sw_if_indices[i] == sw_if_index)
3776         {
3777           if (is_del)
3778             {
3779               /* if have address remove it */
3780               if (first_int_addr)
3781                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3782               else
3783                 {
3784                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3785                     {
3786                       rp = sm->to_resolve + j;
3787                       if (rp->sw_if_index == sw_if_index)
3788                         vec_add1 (indices_to_delete, j);
3789                     }
3790                   if (vec_len (indices_to_delete))
3791                     {
3792                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3793                         vec_del1 (sm->to_resolve, j);
3794                       vec_free (indices_to_delete);
3795                     }
3796                 }
3797               if (twice_nat)
3798                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3799               else
3800                 vec_del1 (sm->auto_add_sw_if_indices, i);
3801             }
3802           else
3803             return VNET_API_ERROR_VALUE_EXIST;
3804
3805           return 0;
3806         }
3807     }
3808
3809   if (is_del)
3810     return VNET_API_ERROR_NO_SUCH_ENTRY;
3811
3812   /* add to the auto-address list */
3813   if (twice_nat)
3814     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3815   else
3816     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3817
3818   /* If the address is already bound - or static - add it now */
3819   if (first_int_addr)
3820     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3821
3822   return 0;
3823 }
3824
3825 int
3826 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3827                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3828                       u32 vrf_id, int is_in)
3829 {
3830   ip4_header_t ip;
3831   clib_bihash_kv_16_8_t kv, value;
3832   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3833   snat_session_t *s;
3834   snat_main_per_thread_data_t *tsm;
3835
3836   if (!sm->endpoint_dependent)
3837     return VNET_API_ERROR_FEATURE_DISABLED;
3838
3839   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3840   if (sm->num_workers > 1)
3841     tsm =
3842       vec_elt_at_index (sm->per_thread_data,
3843                         sm->worker_in2out_cb (&ip, fib_index, 0));
3844   else
3845     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3846
3847   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
3848   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3849     {
3850       return VNET_API_ERROR_NO_SUCH_ENTRY;
3851     }
3852
3853   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3854     return VNET_API_ERROR_UNSPECIFIED;
3855   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3856   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3857   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3858   return 0;
3859 }
3860
3861 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3862                                  vlib_node_runtime_t * node,
3863                                  vlib_frame_t * frame)
3864 {
3865   return 0;
3866 }
3867
3868 /* *INDENT-OFF* */
3869 VLIB_REGISTER_NODE (nat_default_node) = {
3870   .name = "nat-default",
3871   .vector_size = sizeof (u32),
3872   .format_trace = 0,
3873   .type = VLIB_NODE_TYPE_INTERNAL,
3874   .n_errors = 0,
3875   .n_next_nodes = NAT_N_NEXT,
3876   .next_nodes = {
3877     [NAT_NEXT_DROP] = "error-drop",
3878     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3879     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3880     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3881     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3882     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3883     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3884     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3885     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3886     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3887   },
3888 };
3889 /* *INDENT-ON* */
3890
3891 void
3892 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3893 {
3894   f->l3_csum_delta = 0;
3895   f->l4_csum_delta = 0;
3896   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3897       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3898     {
3899       f->l3_csum_delta =
3900         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3901       f->l3_csum_delta =
3902         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3903     }
3904   else
3905     {
3906       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3907     }
3908   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3909       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3910     {
3911       f->l3_csum_delta =
3912         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3913       f->l3_csum_delta =
3914         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3915     }
3916   else
3917     {
3918       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3919     }
3920   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3921     {
3922       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3923       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3924     }
3925   else
3926     {
3927       f->rewrite.sport = f->match.sport;
3928     }
3929   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3930     {
3931       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3932       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3933     }
3934   else
3935     {
3936       f->rewrite.dport = f->match.dport;
3937     }
3938   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3939       f->rewrite.icmp_id != f->match.icmp_id)
3940     {
3941       f->l4_csum_delta =
3942         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3943       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.icmp_id);
3944     }
3945   else
3946     {
3947       f->rewrite.icmp_id = f->match.icmp_id;
3948     }
3949   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3950     {
3951     }
3952   else
3953     {
3954       f->rewrite.fib_index = f->match.fib_index;
3955     }
3956 }
3957
3958 static_always_inline int nat_6t_flow_icmp_translate (snat_main_t *sm,
3959                                                      vlib_buffer_t *b,
3960                                                      ip4_header_t *ip,
3961                                                      nat_6t_flow_t *f);
3962
3963 static_always_inline void
3964 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3965                            nat_6t_flow_t *f, nat_protocol_t proto,
3966                            int is_icmp_inner_ip4)
3967 {
3968   udp_header_t *udp = ip4_next_header (ip);
3969   tcp_header_t *tcp = (tcp_header_t *) udp;
3970
3971   if ((NAT_PROTOCOL_TCP == proto || NAT_PROTOCOL_UDP == proto) &&
3972       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3973     {
3974       if (!is_icmp_inner_ip4)
3975         { // regular case
3976           ip->src_address = f->rewrite.saddr;
3977           ip->dst_address = f->rewrite.daddr;
3978           udp->src_port = f->rewrite.sport;
3979           udp->dst_port = f->rewrite.dport;
3980         }
3981       else
3982         { // icmp inner ip4 - reversed saddr/daddr
3983           ip->src_address = f->rewrite.daddr;
3984           ip->dst_address = f->rewrite.saddr;
3985           udp->src_port = f->rewrite.dport;
3986           udp->dst_port = f->rewrite.sport;
3987         }
3988
3989       if (NAT_PROTOCOL_TCP == proto)
3990         {
3991           ip_csum_t tcp_sum = tcp->checksum;
3992           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3993           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3994           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3995           tcp->checksum = ip_csum_fold (tcp_sum);
3996         }
3997       else if (proto == NAT_PROTOCOL_UDP && udp->checksum)
3998         {
3999           ip_csum_t udp_sum = udp->checksum;
4000           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
4001           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
4002           udp->checksum = ip_csum_fold (udp_sum);
4003         }
4004     }
4005   else
4006     {
4007       if (!is_icmp_inner_ip4)
4008         { // regular case
4009           ip->src_address = f->rewrite.saddr;
4010           ip->dst_address = f->rewrite.daddr;
4011         }
4012       else
4013         { // icmp inner ip4 - reversed saddr/daddr
4014           ip->src_address = f->rewrite.daddr;
4015           ip->dst_address = f->rewrite.saddr;
4016         }
4017     }
4018
4019   ip_csum_t ip_sum = ip->checksum;
4020   ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
4021   ip->checksum = ip_csum_fold (ip_sum);
4022   ASSERT (ip->checksum == ip4_header_checksum (ip));
4023 }
4024
4025 static_always_inline int
4026 nat_6t_flow_icmp_translate (snat_main_t *sm, vlib_buffer_t *b,
4027                             ip4_header_t *ip, nat_6t_flow_t *f)
4028 {
4029   if (IP_PROTOCOL_ICMP != ip->protocol)
4030     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4031
4032   icmp46_header_t *icmp = ip4_next_header (ip);
4033   icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
4034
4035   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
4036     {
4037       if (icmp->checksum == 0)
4038         icmp->checksum = 0xffff;
4039
4040       if (!icmp_type_is_error_message (icmp->type))
4041         {
4042           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
4043               (f->rewrite.icmp_id != echo->identifier))
4044             {
4045               ip_csum_t sum = icmp->checksum;
4046               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
4047                                     icmp_echo_header_t,
4048                                     identifier /* changed member */);
4049               echo->identifier = f->rewrite.icmp_id;
4050               icmp->checksum = ip_csum_fold (sum);
4051             }
4052         }
4053       else
4054         {
4055           // errors are not fragmented
4056           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
4057
4058           if (!ip4_header_checksum_is_valid (inner_ip))
4059             {
4060               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4061             }
4062
4063           nat_protocol_t inner_proto =
4064             ip_proto_to_nat_proto (inner_ip->protocol);
4065
4066           ip_csum_t icmp_sum = icmp->checksum;
4067
4068           switch (inner_proto)
4069             {
4070             case NAT_PROTOCOL_UDP:
4071             case NAT_PROTOCOL_TCP:
4072               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
4073                                          1 /* is_icmp_inner_ip4 */);
4074               icmp_sum = ip_csum_sub_even (icmp_sum, f->l3_csum_delta);
4075               icmp->checksum = ip_csum_fold (icmp_sum);
4076               break;
4077             case NAT_PROTOCOL_ICMP:
4078               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
4079                 {
4080                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
4081                   icmp_echo_header_t *inner_echo =
4082                     (icmp_echo_header_t *) (inner_icmp + 1);
4083                   if (f->rewrite.icmp_id != inner_echo->identifier)
4084                     {
4085                       ip_csum_t sum = icmp->checksum;
4086                       sum = ip_csum_update (
4087                         sum, inner_echo->identifier, f->rewrite.icmp_id,
4088                         icmp_echo_header_t, identifier /* changed member */);
4089                       icmp->checksum = ip_csum_fold (sum);
4090                       ip_csum_t inner_sum = inner_icmp->checksum;
4091                       inner_sum = ip_csum_update (
4092                         sum, inner_echo->identifier, f->rewrite.icmp_id,
4093                         icmp_echo_header_t, identifier /* changed member */);
4094                       inner_icmp->checksum = ip_csum_fold (inner_sum);
4095                       inner_echo->identifier = f->rewrite.icmp_id;
4096                     }
4097                 }
4098               break;
4099             default:
4100               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
4101               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4102             }
4103         }
4104     }
4105   return NAT_ED_TRNSL_ERR_SUCCESS;
4106 }
4107
4108 nat_translation_error_e
4109 nat_6t_flow_buf_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
4110                            nat_6t_flow_t *f, nat_protocol_t proto,
4111                            int is_output_feature)
4112 {
4113   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
4114     {
4115       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
4116     }
4117
4118   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */);
4119
4120   if (NAT_PROTOCOL_ICMP == proto)
4121     {
4122       return nat_6t_flow_icmp_translate (sm, b, ip, f);
4123     }
4124
4125   return NAT_ED_TRNSL_ERR_SUCCESS;
4126 }
4127
4128 u8 *
4129 format_nat_6t (u8 *s, va_list *args)
4130 {
4131   nat_6t_t *t = va_arg (*args, nat_6t_t *);
4132
4133   s = format (s, "saddr %U sport %u daddr %U dport %u proto %U fib_idx %u",
4134               format_ip4_address, t->saddr.as_u8,
4135               clib_net_to_host_u16 (t->sport), format_ip4_address,
4136               t->daddr.as_u8, clib_net_to_host_u16 (t->dport),
4137               format_ip_protocol, t->proto, t->fib_index);
4138   return s;
4139 }
4140
4141 u8 *
4142 format_nat_ed_translation_error (u8 *s, va_list *args)
4143 {
4144   nat_translation_error_e e = va_arg (*args, nat_translation_error_e);
4145
4146   switch (e)
4147     {
4148     case NAT_ED_TRNSL_ERR_SUCCESS:
4149       s = format (s, "success");
4150       break;
4151     case NAT_ED_TRNSL_ERR_TRANSLATION_FAILED:
4152       s = format (s, "translation-failed");
4153       break;
4154     case NAT_ED_TRNSL_ERR_FLOW_MISMATCH:
4155       s = format (s, "flow-mismatch");
4156       break;
4157     }
4158   return s;
4159 }
4160
4161 u8 *
4162 format_nat_6t_flow (u8 *s, va_list *args)
4163 {
4164   nat_6t_flow_t *f = va_arg (*args, nat_6t_flow_t *);
4165
4166   s = format (s, "match: %U ", format_nat_6t, &f->match);
4167   int r = 0;
4168   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE)
4169     {
4170       s = format (s, "rewrite: saddr %U ", format_ip4_address,
4171                   f->rewrite.saddr.as_u8);
4172       r = 1;
4173     }
4174   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE)
4175     {
4176       if (!r)
4177         {
4178           s = format (s, "rewrite: ");
4179           r = 1;
4180         }
4181       s = format (s, "sport %u ", clib_net_to_host_u16 (f->rewrite.sport));
4182     }
4183   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE)
4184     {
4185       if (!r)
4186         {
4187           s = format (s, "rewrite: ");
4188           r = 1;
4189         }
4190       s = format (s, "daddr %U ", format_ip4_address, f->rewrite.daddr.as_u8);
4191     }
4192   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE)
4193     {
4194       if (!r)
4195         {
4196           s = format (s, "rewrite: ");
4197           r = 1;
4198         }
4199       s = format (s, "dport %u ", clib_net_to_host_u16 (f->rewrite.dport));
4200     }
4201   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
4202     {
4203       if (!r)
4204         {
4205           s = format (s, "rewrite: ");
4206           r = 1;
4207         }
4208       s = format (s, "icmp-id %u ", clib_net_to_host_u16 (f->rewrite.icmp_id));
4209     }
4210   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
4211     {
4212       if (!r)
4213         {
4214           s = format (s, "rewrite: ");
4215           r = 1;
4216         }
4217       s = format (s, "txfib %u ", f->rewrite.fib_index);
4218     }
4219   return s;
4220 }
4221
4222 /*
4223  * fd.io coding-style-patch-verification: ON
4224  *
4225  * Local Variables:
4226  * eval: (c-set-style "gnu")
4227  * End:
4228  */