src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/lib/ipfix_logging.h>
  25 #include <nat/lib/nat_syslog.h>
  26 #include <nat/nat_inlines.h>
  27 #include <nat/nat44/inlines.h>
  28 #include <nat/nat_affinity.h>
  29 #include <vnet/fib/fib_table.h>
  30 #include <vnet/fib/ip4_fib.h>
  31 #include <vnet/ip/reass/ip4_sv_reass.h>
  32 #include <vppinfra/bihash_16_8.h>
  33 #include <nat/nat44/ed_inlines.h>
  34 #include <vnet/ip/ip_table.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_ha.h>
  37 #include <nat/nat44-ei/nat44_ei.h>
  38
  39 #include <vpp/app/version.h>
  40 #include <nat/lib/nat_inlines.h>
  41
  42 snat_main_t snat_main;
  43
  44 #define skip_if_disabled()                                                    \
  45   do                                                                          \
  46     {                                                                         \
  47       snat_main_t *sm = &snat_main;                                           \
  48       if (PREDICT_FALSE (!sm->enabled))                                       \
  49         return;                                                               \
  50     }                                                                         \
  51   while (0)
  52
  53 #define fail_if_enabled()                                                     \
  54   do                                                                          \
  55     {                                                                         \
  56       snat_main_t *sm = &snat_main;                                           \
  57       if (PREDICT_FALSE (sm->enabled))                                        \
  58         {                                                                     \
  59           nat_log_err ("plugin enabled");                                     \
  60           return 1;                                                           \
  61         }                                                                     \
  62     }                                                                         \
  63   while (0)
  64
  65 #define fail_if_disabled()                                                    \
  66   do                                                                          \
  67     {                                                                         \
  68       snat_main_t *sm = &snat_main;                                           \
  69       if (PREDICT_FALSE (!sm->enabled))                                       \
  70         {                                                                     \
  71           nat_log_err ("plugin disabled");                                    \
  72           return 1;                                                           \
  73         }                                                                     \
  74     }                                                                         \
  75   while (0)
  76
  77 /* *INDENT-OFF* */
  78 /* Hook up input features */
  79 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat-pre-in2out",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  83                                "ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat-pre-out2in",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  89                                "ip4-dhcp-client-detect",
  90                                "ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-in2out-worker-handoff",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  96 };
  97 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-out2in-worker-handoff",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 101                                "ip4-dhcp-client-detect"),
 102 };
 103 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-in2out",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-out2in",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 112                                "ip4-dhcp-client-detect"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-classify",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 118 };
 119 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-ed-in2out",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 123 };
 124 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-ed-out2in",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 128                                "ip4-dhcp-client-detect"),
 129 };
 130 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 131   .arc_name = "ip4-unicast",
 132   .node_name = "nat44-ed-classify",
 133   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 134 };
 135 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 136   .arc_name = "ip4-unicast",
 137   .node_name = "nat44-handoff-classify",
 138   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 141   .arc_name = "ip4-unicast",
 142   .node_name = "nat44-in2out-fast",
 143   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 144 };
 145 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 146   .arc_name = "ip4-unicast",
 147   .node_name = "nat44-out2in-fast",
 148   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 149                                "ip4-dhcp-client-detect"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 152   .arc_name = "ip4-unicast",
 153   .node_name = "nat44-hairpin-dst",
 154   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 155 };
 156
 157 /* Hook up output features */
 158 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 159   .arc_name = "ip4-output",
 160   .node_name = "nat44-in2out-output",
 161   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 162 };
 163 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 164   .arc_name = "ip4-output",
 165   .node_name = "nat44-in2out-output-worker-handoff",
 166   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 167 };
 168 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 169   .arc_name = "ip4-output",
 170   .node_name = "nat44-hairpin-src",
 171   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 172 };
 173 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 174   .arc_name = "ip4-output",
 175   .node_name = "nat-pre-in2out-output",
 176   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 177   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 178 };
 179 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 180   .arc_name = "ip4-output",
 181   .node_name = "nat44-ed-in2out-output",
 182   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 183   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 184 };
 185
 186 /* Hook up ip4-local features */
 187 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 188 {
 189   .arc_name = "ip4-local",
 190   .node_name = "nat44-hairpinning",
 191   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 192 };
 193
 194 VLIB_PLUGIN_REGISTER () = {
 195     .version = VPP_BUILD_VER,
 196     .description = "Network Address Translation (NAT)",
 197 };
 198 /* *INDENT-ON* */
 199
 200 static void nat44_ed_db_init (u32 translations, u32 translation_buckets,
 201                               u32 user_buckets);
 202
 203 static void nat44_ed_db_free ();
 204
 205 static u32
 206 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
 207                                u32 rx_fib_index, u8 is_output);
 208
 209 static u32
 210 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
 211                                u8 is_output);
 212
 213 u32 nat_calc_bihash_buckets (u32 n_elts);
 214
 215 u8 *
 216 format_session_kvp (u8 * s, va_list * args)
 217 {
 218   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 219
 220   s = format (s, "%U session-index %llu", format_snat_key, v->key, v->value);
 221
 222   return s;
 223 }
 224
 225 u8 *
 226 format_static_mapping_kvp (u8 * s, va_list * args)
 227 {
 228   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 229
 230   s = format (s, "%U static-mapping-index %llu",
 231               format_snat_key, v->key, v->value);
 232
 233   return s;
 234 }
 235
 236 u8 *
 237 format_user_kvp (u8 * s, va_list * args)
 238 {
 239   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 240   snat_user_key_t k;
 241
 242   k.as_u64 = v->key;
 243
 244   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
 245               k.fib_index, v->value);
 246
 247   return s;
 248 }
 249
 250 u8 *
 251 format_ed_session_kvp (u8 * s, va_list * args)
 252 {
 253   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 254
 255   u8 proto;
 256   u16 r_port, l_port;
 257   ip4_address_t l_addr, r_addr;
 258   u32 fib_index;
 259
 260   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 261   s = format (s,
 262               "local %U:%d remote %U:%d proto %U fib %d thread-index %u "
 263               "session-index %u",
 264               format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 265               format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 266               format_ip_protocol, proto, fib_index,
 267               ed_value_get_thread_index (v), ed_value_get_session_index (v));
 268
 269   return s;
 270 }
 271
 272 void
 273 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 274                        u8 is_ha)
 275 {
 276   clib_bihash_kv_8_8_t kv;
 277   snat_main_per_thread_data_t *tsm =
 278     vec_elt_at_index (sm->per_thread_data, thread_index);
 279
 280   if (is_ed_session (s))
 281     {
 282       per_vrf_sessions_unregister_session (s, thread_index);
 283
 284       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 285         nat_elog_warn ("flow hash del failed");
 286
 287       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 288         nat_elog_warn ("flow hash del failed");
 289     }
 290
 291   if (is_fwd_bypass_session (s))
 292     {
 293       return;
 294     }
 295
 296   /* session lookup tables */
 297   if (is_ed_session (s))
 298     {
 299       if (is_affinity_sessions (s))
 300         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 301                              s->nat_proto, s->out2in.port);
 302
 303       if (!is_ha)
 304         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 305                                &s->in2out.addr, s->in2out.port,
 306                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 307                                &s->out2in.addr, s->out2in.port,
 308                                &s->ext_host_addr, s->ext_host_port,
 309                                s->nat_proto, is_twice_nat_session (s));
 310     }
 311   else
 312     {
 313       init_nat_i2o_k (&kv, s);
 314       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 315         nat_elog_warn ("in2out key del failed");
 316       init_nat_o2i_k (&kv, s);
 317       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 318         nat_elog_warn ("out2in key del failed");
 319
 320       if (!is_ha)
 321         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 322                                  &s->in2out.addr, s->in2out.port,
 323                                  &s->out2in.addr, s->out2in.port,
 324                                  s->nat_proto);
 325     }
 326
 327   if (snat_is_unk_proto_session (s))
 328     return;
 329
 330   if (!is_ha)
 331     {
 332       /* log NAT event */
 333       nat_ipfix_logging_nat44_ses_delete (thread_index,
 334                                           s->in2out.addr.as_u32,
 335                                           s->out2in.addr.as_u32,
 336                                           s->nat_proto,
 337                                           s->in2out.port,
 338                                           s->out2in.port,
 339                                           s->in2out.fib_index);
 340
 341       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 342                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 343                    thread_index);
 344     }
 345
 346   /* Twice NAT address and port for external host */
 347   if (is_twice_nat_session (s))
 348     {
 349       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 350                                           thread_index,
 351                                           &s->ext_host_nat_addr,
 352                                           s->ext_host_nat_port, s->nat_proto);
 353     }
 354
 355   if (snat_is_session_static (s))
 356     return;
 357
 358   snat_free_outside_address_and_port (sm->addresses, thread_index,
 359                                       &s->out2in.addr, s->out2in.port,
 360                                       s->nat_proto);
 361 }
 362
 363 snat_user_t *
 364 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 365                         u32 thread_index)
 366 {
 367   snat_user_t *u = 0;
 368   snat_user_key_t user_key;
 369   clib_bihash_kv_8_8_t kv, value;
 370   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 371   dlist_elt_t *per_user_list_head_elt;
 372
 373   user_key.addr.as_u32 = addr->as_u32;
 374   user_key.fib_index = fib_index;
 375   kv.key = user_key.as_u64;
 376
 377   /* Ever heard of the "user" = src ip4 address before? */
 378   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 379     {
 380       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 381         {
 382           vlib_increment_simple_counter (&sm->user_limit_reached,
 383                                          thread_index, 0, 1);
 384           nat_elog_warn ("maximum user limit reached");
 385           return NULL;
 386         }
 387       /* no, make a new one */
 388       pool_get (tsm->users, u);
 389       clib_memset (u, 0, sizeof (*u));
 390
 391       u->addr.as_u32 = addr->as_u32;
 392       u->fib_index = fib_index;
 393
 394       pool_get (tsm->list_pool, per_user_list_head_elt);
 395
 396       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 397         tsm->list_pool;
 398
 399       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 400
 401       kv.value = u - tsm->users;
 402
 403       /* add user */
 404       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 405         {
 406           nat_elog_warn ("user_hash key add failed");
 407           nat44_delete_user_with_no_session (sm, u, thread_index);
 408           return NULL;
 409         }
 410
 411       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 412                                pool_elts (tsm->users));
 413     }
 414   else
 415     {
 416       u = pool_elt_at_index (tsm->users, value.value);
 417     }
 418
 419   return u;
 420 }
 421
 422 // only NAT EI
 423 snat_session_t *
 424 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 425                               u32 thread_index, f64 now)
 426 {
 427   snat_session_t *s;
 428   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 429   u32 oldest_per_user_translation_list_index, session_index;
 430   dlist_elt_t *oldest_per_user_translation_list_elt;
 431   dlist_elt_t *per_user_translation_list_elt;
 432
 433   /* Over quota? Recycle the least recently used translation */
 434   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 435     {
 436       oldest_per_user_translation_list_index =
 437         clib_dlist_remove_head (tsm->list_pool,
 438                                 u->sessions_per_user_list_head_index);
 439
 440       ASSERT (oldest_per_user_translation_list_index != ~0);
 441
 442       /* Add it back to the end of the LRU list */
 443       clib_dlist_addtail (tsm->list_pool,
 444                           u->sessions_per_user_list_head_index,
 445                           oldest_per_user_translation_list_index);
 446       /* Get the list element */
 447       oldest_per_user_translation_list_elt =
 448         pool_elt_at_index (tsm->list_pool,
 449                            oldest_per_user_translation_list_index);
 450
 451       /* Get the session index from the list element */
 452       session_index = oldest_per_user_translation_list_elt->value;
 453
 454       /* Get the session */
 455       s = pool_elt_at_index (tsm->sessions, session_index);
 456
 457       // TODO: ONLY EI version should be called
 458       nat_free_session_data (sm, s, thread_index, 0);
 459       if (snat_is_session_static (s))
 460         u->nstaticsessions--;
 461       else
 462         u->nsessions--;
 463       s->flags = 0;
 464       s->total_bytes = 0;
 465       s->total_pkts = 0;
 466       s->state = 0;
 467       s->ext_host_addr.as_u32 = 0;
 468       s->ext_host_port = 0;
 469       s->ext_host_nat_addr.as_u32 = 0;
 470       s->ext_host_nat_port = 0;
 471     }
 472   else
 473     {
 474       pool_get (tsm->sessions, s);
 475       clib_memset (s, 0, sizeof (*s));
 476
 477       /* Create list elts */
 478       pool_get (tsm->list_pool, per_user_translation_list_elt);
 479       clib_dlist_init (tsm->list_pool,
 480                        per_user_translation_list_elt - tsm->list_pool);
 481
 482       per_user_translation_list_elt->value = s - tsm->sessions;
 483       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 484       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 485
 486       clib_dlist_addtail (tsm->list_pool,
 487                           s->per_user_list_head_index,
 488                           per_user_translation_list_elt - tsm->list_pool);
 489
 490       s->user_index = u - tsm->users;
 491       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 492                                pool_elts (tsm->sessions));
 493     }
 494
 495   s->ha_last_refreshed = now;
 496
 497   return s;
 498 }
 499
 500 void
 501 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 502                           int is_add)
 503 {
 504   snat_main_t *sm = &snat_main;
 505   fib_prefix_t prefix = {
 506     .fp_len = p_len,
 507     .fp_proto = FIB_PROTOCOL_IP4,
 508     .fp_addr = {
 509                 .ip4.as_u32 = addr->as_u32,
 510                 },
 511   };
 512   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 513
 514   if (is_add)
 515     fib_table_entry_update_one_path (fib_index,
 516                                      &prefix,
 517                                      sm->fib_src_low,
 518                                      (FIB_ENTRY_FLAG_CONNECTED |
 519                                       FIB_ENTRY_FLAG_LOCAL |
 520                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 521                                      DPO_PROTO_IP4,
 522                                      NULL,
 523                                      sw_if_index,
 524                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 525   else
 526     fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 527 }
 528
 529 int
 530 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 531                   u8 twice_nat)
 532 {
 533   snat_address_t *ap;
 534   snat_interface_t *i;
 535   vlib_thread_main_t *tm = vlib_get_thread_main ();
 536
 537   if (twice_nat && !sm->endpoint_dependent)
 538     {
 539       nat_log_err ("unsupported");
 540       return VNET_API_ERROR_UNSUPPORTED;
 541     }
 542
 543   /* Check if address already exists */
 544   /* *INDENT-OFF* */
 545   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 546     {
 547       if (ap->addr.as_u32 == addr->as_u32)
 548         {
 549           nat_log_err ("address exist");
 550           return VNET_API_ERROR_VALUE_EXIST;
 551         }
 552     }
 553   /* *INDENT-ON* */
 554
 555   if (twice_nat)
 556     vec_add2 (sm->twice_nat_addresses, ap, 1);
 557   else
 558     vec_add2 (sm->addresses, ap, 1);
 559
 560   ap->addr = *addr;
 561   if (vrf_id != ~0)
 562     ap->fib_index =
 563       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 564                                          sm->fib_src_low);
 565   else
 566     ap->fib_index = ~0;
 567
 568   /* *INDENT-OFF* */
 569   #define _(N, i, n, s) \
 570     clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 571     ap->busy_##n##_ports = 0; \
 572     ap->busy_##n##_ports_per_thread = 0;\
 573     vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 574     foreach_nat_protocol
 575   #undef _
 576   /* *INDENT-ON* */
 577
 578   if (twice_nat)
 579     return 0;
 580
 581   /* Add external address to FIB */
 582   /* *INDENT-OFF* */
 583   pool_foreach (i, sm->interfaces)
 584    {
 585     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 586       continue;
 587
 588     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 589     break;
 590   }
 591   pool_foreach (i, sm->output_feature_interfaces)
 592    {
 593     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 594       continue;
 595
 596     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 597     break;
 598   }
 599   /* *INDENT-ON* */
 600
 601   return 0;
 602 }
 603
 604 static int
 605 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 606 {
 607   snat_static_mapping_t *m;
 608   /* *INDENT-OFF* */
 609   pool_foreach (m, sm->static_mappings)
 610    {
 611       if (is_addr_only_static_mapping (m) ||
 612           is_out2in_only_static_mapping (m) ||
 613           is_identity_static_mapping (m))
 614         continue;
 615       if (m->external_addr.as_u32 == addr.as_u32)
 616         return 1;
 617   }
 618   /* *INDENT-ON* */
 619
 620   return 0;
 621 }
 622
 623 static void
 624 snat_add_static_mapping_when_resolved (snat_main_t *sm, ip4_address_t l_addr,
 625                                        u16 l_port, u32 sw_if_index, u16 e_port,
 626                                        u32 vrf_id, nat_protocol_t proto,
 627                                        int addr_only, u8 *tag, int twice_nat,
 628                                        int out2in_only, int identity_nat,
 629                                        ip4_address_t pool_addr, int exact)
 630 {
 631   snat_static_map_resolve_t *rp;
 632
 633   vec_add2 (sm->to_resolve, rp, 1);
 634   rp->l_addr.as_u32 = l_addr.as_u32;
 635   rp->l_port = l_port;
 636   rp->sw_if_index = sw_if_index;
 637   rp->e_port = e_port;
 638   rp->vrf_id = vrf_id;
 639   rp->proto = proto;
 640   rp->addr_only = addr_only;
 641   rp->twice_nat = twice_nat;
 642   rp->out2in_only = out2in_only;
 643   rp->identity_nat = identity_nat;
 644   rp->tag = vec_dup (tag);
 645   rp->pool_addr = pool_addr;
 646   rp->exact = exact;
 647 }
 648
 649 u32
 650 get_thread_idx_by_port (u16 e_port)
 651 {
 652   snat_main_t *sm = &snat_main;
 653   u32 thread_idx = sm->num_workers;
 654   if (sm->num_workers > 1)
 655     {
 656       thread_idx =
 657         sm->first_worker_index +
 658         sm->workers[(e_port - 1024) / sm->port_per_thread];
 659     }
 660   return thread_idx;
 661 }
 662
 663 void
 664 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 665                                     snat_main_per_thread_data_t * tsm,
 666                                     ip4_address_t l_addr,
 667                                     u16 l_port,
 668                                     u8 protocol,
 669                                     u32 fib_index, int addr_only,
 670                                     ip4_address_t e_addr, u16 e_port)
 671 {
 672   snat_session_t *s;
 673   u32 *indexes_to_free = NULL;
 674   /* *INDENT-OFF* */
 675   pool_foreach (s, tsm->sessions) {
 676     if (s->in2out.fib_index != fib_index ||
 677         s->in2out.addr.as_u32 != l_addr.as_u32)
 678       {
 679         continue;
 680       }
 681     if (!addr_only)
 682       {
 683         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 684             s->out2in.port != e_port ||
 685             s->in2out.port != l_port ||
 686             s->nat_proto != protocol)
 687           continue;
 688       }
 689
 690     if (is_lb_session (s))
 691       continue;
 692     if (!snat_is_session_static (s))
 693       continue;
 694     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 695     vec_add1 (indexes_to_free, s - tsm->sessions);
 696     if (!addr_only)
 697       break;
 698   }
 699   /* *INDENT-ON* */
 700   u32 *ses_index;
 701   vec_foreach (ses_index, indexes_to_free)
 702   {
 703     s = pool_elt_at_index (tsm->sessions, *ses_index);
 704     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 705   }
 706   vec_free (indexes_to_free);
 707 }
 708
 709 static_always_inline int
 710 nat44_ed_add_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 711                                  u16 l_port, u16 e_port, u32 vrf_id,
 712                                  int addr_only, u32 sw_if_index,
 713                                  nat_protocol_t proto, int is_add,
 714                                  twice_nat_type_t twice_nat, u8 out2in_only,
 715                                  u8 *tag, u8 identity_nat,
 716                                  ip4_address_t pool_addr, int exact)
 717 {
 718   snat_main_t *sm = &snat_main;
 719   snat_static_mapping_t *m;
 720   clib_bihash_kv_8_8_t kv, value;
 721   snat_address_t *a = 0;
 722   u32 fib_index = ~0;
 723   snat_interface_t *interface;
 724   int i;
 725   snat_main_per_thread_data_t *tsm;
 726   snat_user_key_t u_key;
 727   snat_user_t *u;
 728   dlist_elt_t *head, *elt;
 729   u32 elt_index, head_index;
 730   u32 ses_index;
 731   u64 user_index;
 732   snat_session_t *s;
 733   snat_static_map_resolve_t *rp, *rp_match = 0;
 734   nat44_lb_addr_port_t *local;
 735   u32 find = ~0;
 736
 737   if (!sm->endpoint_dependent)
 738     {
 739       if (twice_nat || out2in_only)
 740         return VNET_API_ERROR_UNSUPPORTED;
 741     }
 742
 743   /* If the external address is a specific interface address */
 744   if (sw_if_index != ~0)
 745     {
 746       ip4_address_t *first_int_addr;
 747
 748       for (i = 0; i < vec_len (sm->to_resolve); i++)
 749         {
 750           rp = sm->to_resolve + i;
 751           if (rp->sw_if_index != sw_if_index ||
 752               rp->l_addr.as_u32 != l_addr.as_u32 ||
 753               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 754             continue;
 755
 756           if (!addr_only)
 757             {
 758               if ((rp->l_port != l_port && rp->e_port != e_port)
 759                   || rp->proto != proto)
 760                 continue;
 761             }
 762
 763           rp_match = rp;
 764           break;
 765         }
 766
 767       /* Might be already set... */
 768       first_int_addr = ip4_interface_first_address
 769         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 770
 771       if (is_add)
 772         {
 773           if (rp_match)
 774             return VNET_API_ERROR_VALUE_EXIST;
 775
 776           snat_add_static_mapping_when_resolved (
 777             sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto, addr_only,
 778             tag, twice_nat, out2in_only, identity_nat, pool_addr, exact);
 779
 780           /* DHCP resolution required? */
 781           if (first_int_addr == 0)
 782             {
 783               return 0;
 784             }
 785           else
 786             {
 787               e_addr.as_u32 = first_int_addr->as_u32;
 788               /* Identity mapping? */
 789               if (l_addr.as_u32 == 0)
 790                 l_addr.as_u32 = e_addr.as_u32;
 791             }
 792         }
 793       else
 794         {
 795           if (!rp_match)
 796             return VNET_API_ERROR_NO_SUCH_ENTRY;
 797
 798           vec_del1 (sm->to_resolve, i);
 799
 800           if (first_int_addr)
 801             {
 802               e_addr.as_u32 = first_int_addr->as_u32;
 803               /* Identity mapping? */
 804               if (l_addr.as_u32 == 0)
 805                 l_addr.as_u32 = e_addr.as_u32;
 806             }
 807           else
 808             return 0;
 809         }
 810     }
 811
 812   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 813   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 814     m = 0;
 815   else
 816     m = pool_elt_at_index (sm->static_mappings, value.value);
 817
 818   if (is_add)
 819     {
 820       if (m)
 821         {
 822           if (is_identity_static_mapping (m))
 823             {
 824               /* *INDENT-OFF* */
 825               pool_foreach (local, m->locals)
 826                {
 827                 if (local->vrf_id == vrf_id)
 828                   return VNET_API_ERROR_VALUE_EXIST;
 829               }
 830               /* *INDENT-ON* */
 831               pool_get (m->locals, local);
 832               local->vrf_id = vrf_id;
 833               local->fib_index =
 834                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 835                                                    sm->fib_src_low);
 836               init_nat_kv (&kv, m->local_addr, m->local_port,
 837                            local->fib_index, m->proto,
 838                            m - sm->static_mappings);
 839               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 840               return 0;
 841             }
 842           else
 843             return VNET_API_ERROR_VALUE_EXIST;
 844         }
 845
 846       if (twice_nat && addr_only)
 847         return VNET_API_ERROR_UNSUPPORTED;
 848
 849       /* Convert VRF id to FIB index */
 850       if (vrf_id != ~0)
 851         fib_index =
 852           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 853                                              sm->fib_src_low);
 854       /* If not specified use inside VRF id from SNAT plugin startup config */
 855       else
 856         {
 857           fib_index = sm->inside_fib_index;
 858           vrf_id = sm->inside_vrf_id;
 859           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 860         }
 861
 862       if (!(out2in_only || identity_nat))
 863         {
 864           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
 865                       addr_only ? 0 : proto);
 866           if (!clib_bihash_search_8_8
 867               (&sm->static_mapping_by_local, &kv, &value))
 868             return VNET_API_ERROR_VALUE_EXIST;
 869         }
 870
 871       /* Find external address in allocated addresses and reserve port for
 872          address and port pair mapping when dynamic translations enabled */
 873       if (!(addr_only || sm->static_mapping_only || out2in_only))
 874         {
 875           for (i = 0; i < vec_len (sm->addresses); i++)
 876             {
 877               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 878                 {
 879                   a = sm->addresses + i;
 880                   /* External port must be unused */
 881                   switch (proto)
 882                     {
 883 #define _(N, j, n, s) \
 884                     case NAT_PROTOCOL_##N: \
 885                       if (a->busy_##n##_port_refcounts[e_port]) \
 886                         return VNET_API_ERROR_INVALID_VALUE; \
 887                       ++a->busy_##n##_port_refcounts[e_port]; \
 888                       if (e_port > 1024) \
 889                         { \
 890                           a->busy_##n##_ports++; \
 891                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 892                         } \
 893                       break;
 894                       foreach_nat_protocol
 895 #undef _
 896                     default:
 897                       nat_elog_info ("unknown protocol");
 898                       return VNET_API_ERROR_INVALID_VALUE_2;
 899                     }
 900                   break;
 901                 }
 902             }
 903           /* External address must be allocated */
 904           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 905             {
 906               if (sw_if_index != ~0)
 907                 {
 908                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 909                     {
 910                       rp = sm->to_resolve + i;
 911                       if (rp->addr_only)
 912                         continue;
 913                       if (rp->sw_if_index != sw_if_index &&
 914                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 915                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 916                           rp->e_port != e_port && rp->proto != proto)
 917                         continue;
 918
 919                       vec_del1 (sm->to_resolve, i);
 920                       break;
 921                     }
 922                 }
 923               return VNET_API_ERROR_NO_SUCH_ENTRY;
 924             }
 925         }
 926
 927       pool_get (sm->static_mappings, m);
 928       clib_memset (m, 0, sizeof (*m));
 929       m->tag = vec_dup (tag);
 930       m->local_addr = l_addr;
 931       m->external_addr = e_addr;
 932       m->twice_nat = twice_nat;
 933
 934       if (twice_nat == TWICE_NAT && exact)
 935         {
 936           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
 937           m->pool_addr = pool_addr;
 938         }
 939
 940       if (out2in_only)
 941         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 942       if (addr_only)
 943         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 944       if (identity_nat)
 945         {
 946           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 947           pool_get (m->locals, local);
 948           local->vrf_id = vrf_id;
 949           local->fib_index = fib_index;
 950         }
 951       else
 952         {
 953           m->vrf_id = vrf_id;
 954           m->fib_index = fib_index;
 955         }
 956       if (!addr_only)
 957         {
 958           m->local_port = l_port;
 959           m->external_port = e_port;
 960           m->proto = proto;
 961         }
 962
 963       if (sm->num_workers > 1)
 964         {
 965           ip4_header_t ip = {
 966             .src_address = m->local_addr,
 967           };
 968           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 969           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 970         }
 971       else
 972         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 973
 974       if (!out2in_only)
 975         {
 976           init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
 977                        m - sm->static_mappings);
 978           clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 979         }
 980
 981       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
 982                    m - sm->static_mappings);
 983       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 984
 985       /* Delete dynamic sessions matching local address (+ local port) */
 986       // TODO: based on type of NAT EI/ED
 987       if (!(sm->static_mapping_only))
 988         {
 989           u_key.addr = m->local_addr;
 990           u_key.fib_index = m->fib_index;
 991           kv.key = u_key.as_u64;
 992           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 993             {
 994               user_index = value.value;
 995               u = pool_elt_at_index (tsm->users, user_index);
 996               if (u->nsessions)
 997                 {
 998                   head_index = u->sessions_per_user_list_head_index;
 999                   head = pool_elt_at_index (tsm->list_pool, head_index);
1000                   elt_index = head->next;
1001                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1002                   ses_index = elt->value;
1003                   while (ses_index != ~0)
1004                     {
1005                       s = pool_elt_at_index (tsm->sessions, ses_index);
1006                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1007                       ses_index = elt->value;
1008
1009                       if (snat_is_session_static (s))
1010                         continue;
1011
1012                       if (!addr_only && s->in2out.port != m->local_port)
1013                         continue;
1014
1015                       nat_free_session_data (sm, s,
1016                                              tsm - sm->per_thread_data, 0);
1017                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1018
1019                       if (!addr_only && !sm->endpoint_dependent)
1020                         break;
1021                     }
1022                 }
1023             }
1024         }
1025     }
1026   else
1027     {
1028       if (!m)
1029         {
1030           if (sw_if_index != ~0)
1031             return 0;
1032           else
1033             return VNET_API_ERROR_NO_SUCH_ENTRY;
1034         }
1035
1036       if (identity_nat)
1037         {
1038           if (vrf_id == ~0)
1039             vrf_id = sm->inside_vrf_id;
1040
1041           /* *INDENT-OFF* */
1042           pool_foreach (local, m->locals)
1043            {
1044             if (local->vrf_id == vrf_id)
1045               find = local - m->locals;
1046           }
1047           /* *INDENT-ON* */
1048           if (find == ~0)
1049             return VNET_API_ERROR_NO_SUCH_ENTRY;
1050
1051           local = pool_elt_at_index (m->locals, find);
1052           fib_index = local->fib_index;
1053           pool_put (m->locals, local);
1054         }
1055       else
1056         fib_index = m->fib_index;
1057
1058       /* Free external address port */
1059       if (!(addr_only || sm->static_mapping_only || out2in_only))
1060         {
1061           for (i = 0; i < vec_len (sm->addresses); i++)
1062             {
1063               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1064                 {
1065                   a = sm->addresses + i;
1066                   switch (proto)
1067                     {
1068 #define _(N, j, n, s) \
1069                     case NAT_PROTOCOL_##N: \
1070                       --a->busy_##n##_port_refcounts[e_port]; \
1071                       if (e_port > 1024) \
1072                         { \
1073                           a->busy_##n##_ports--; \
1074                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1075                         } \
1076                       break;
1077                       foreach_nat_protocol
1078 #undef _
1079                     default:
1080                       nat_elog_info ("unknown protocol");
1081                       return VNET_API_ERROR_INVALID_VALUE_2;
1082                     }
1083                   break;
1084                 }
1085             }
1086         }
1087
1088       if (sm->num_workers > 1)
1089         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1090       else
1091         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1092
1093       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1094       if (!out2in_only)
1095         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1096
1097       /* Delete session(s) for static mapping if exist */
1098       if (!(sm->static_mapping_only) ||
1099           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1100         {
1101           if (sm->endpoint_dependent)
1102             {
1103               nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1104                                                   m->local_port, m->proto,
1105                                                   fib_index, addr_only,
1106                                                   e_addr, e_port);
1107             }
1108           else
1109             {
1110               u_key.addr = m->local_addr;
1111               u_key.fib_index = fib_index;
1112               kv.key = u_key.as_u64;
1113               nat44_ei_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1114                                                     e_addr, e_port);
1115             }
1116         }
1117
1118       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1119       if (pool_elts (m->locals))
1120         return 0;
1121
1122       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1123       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1124
1125       vec_free (m->tag);
1126       vec_free (m->workers);
1127       /* Delete static mapping from pool */
1128       pool_put (sm->static_mappings, m);
1129     }
1130
1131   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1132     return 0;
1133
1134   /* Add/delete external address to FIB */
1135   /* *INDENT-OFF* */
1136   pool_foreach (interface, sm->interfaces)
1137    {
1138     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1139       continue;
1140
1141     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1142     break;
1143   }
1144   pool_foreach (interface, sm->output_feature_interfaces)
1145    {
1146     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1147       continue;
1148
1149     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1150     break;
1151   }
1152   /* *INDENT-ON* */
1153
1154   return 0;
1155 }
1156
1157 int
1158 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
1159                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
1160                          u32 sw_if_index, nat_protocol_t proto, int is_add,
1161                          twice_nat_type_t twice_nat, u8 out2in_only, u8 *tag,
1162                          u8 identity_nat, ip4_address_t pool_addr, int exact)
1163 {
1164   snat_main_t *sm = &snat_main;
1165   int rv;
1166
1167   if (sm->endpoint_dependent)
1168     {
1169       rv = nat44_ed_add_del_static_mapping (
1170         l_addr, e_addr, l_port, e_port, vrf_id, addr_only, sw_if_index, proto,
1171         is_add, twice_nat, out2in_only, tag, identity_nat, pool_addr, exact);
1172     }
1173   else
1174     {
1175       rv = nat44_ei_add_del_static_mapping (
1176         l_addr, e_addr, l_port, e_port, proto, sw_if_index, vrf_id, addr_only,
1177         identity_nat, tag, is_add);
1178     }
1179   return rv;
1180 }
1181
1182 int
1183 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1184                                  nat_protocol_t proto,
1185                                  nat44_lb_addr_port_t * locals, u8 is_add,
1186                                  twice_nat_type_t twice_nat, u8 out2in_only,
1187                                  u8 * tag, u32 affinity)
1188 {
1189   snat_main_t *sm = &snat_main;
1190   snat_static_mapping_t *m;
1191   clib_bihash_kv_8_8_t kv, value;
1192   snat_address_t *a = 0;
1193   int i;
1194   nat44_lb_addr_port_t *local;
1195   snat_main_per_thread_data_t *tsm;
1196   snat_session_t *s;
1197   uword *bitmap = 0;
1198
1199   if (!sm->endpoint_dependent)
1200     return VNET_API_ERROR_UNSUPPORTED;
1201
1202   init_nat_k (&kv, e_addr, e_port, 0, proto);
1203   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1204     m = 0;
1205   else
1206     m = pool_elt_at_index (sm->static_mappings, value.value);
1207
1208   if (is_add)
1209     {
1210       if (m)
1211         return VNET_API_ERROR_VALUE_EXIST;
1212
1213       if (vec_len (locals) < 2)
1214         return VNET_API_ERROR_INVALID_VALUE;
1215
1216       /* Find external address in allocated addresses and reserve port for
1217          address and port pair mapping when dynamic translations enabled */
1218       if (!(sm->static_mapping_only || out2in_only))
1219         {
1220           for (i = 0; i < vec_len (sm->addresses); i++)
1221             {
1222               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1223                 {
1224                   a = sm->addresses + i;
1225                   /* External port must be unused */
1226                   switch (proto)
1227                     {
1228 #define _(N, j, n, s) \
1229                     case NAT_PROTOCOL_##N: \
1230                       if (a->busy_##n##_port_refcounts[e_port]) \
1231                         return VNET_API_ERROR_INVALID_VALUE; \
1232                       ++a->busy_##n##_port_refcounts[e_port]; \
1233                       if (e_port > 1024) \
1234                         { \
1235                           a->busy_##n##_ports++; \
1236                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1237                         } \
1238                       break;
1239                       foreach_nat_protocol
1240 #undef _
1241                     default:
1242                       nat_elog_info ("unknown protocol");
1243                       return VNET_API_ERROR_INVALID_VALUE_2;
1244                     }
1245                   break;
1246                 }
1247             }
1248           /* External address must be allocated */
1249           if (!a)
1250             return VNET_API_ERROR_NO_SUCH_ENTRY;
1251         }
1252
1253       pool_get (sm->static_mappings, m);
1254       clib_memset (m, 0, sizeof (*m));
1255       m->tag = vec_dup (tag);
1256       m->external_addr = e_addr;
1257       m->external_port = e_port;
1258       m->proto = proto;
1259       m->twice_nat = twice_nat;
1260       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1261       if (out2in_only)
1262         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1263       m->affinity = affinity;
1264
1265       if (affinity)
1266         m->affinity_per_service_list_head_index =
1267           nat_affinity_get_per_service_list_head_index ();
1268       else
1269         m->affinity_per_service_list_head_index = ~0;
1270
1271       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1272                    m - sm->static_mappings);
1273       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1274         {
1275           nat_elog_err ("static_mapping_by_external key add failed");
1276           return VNET_API_ERROR_UNSPECIFIED;
1277         }
1278
1279       for (i = 0; i < vec_len (locals); i++)
1280         {
1281           locals[i].fib_index =
1282             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1283                                                locals[i].vrf_id,
1284                                                sm->fib_src_low);
1285           if (!out2in_only)
1286             {
1287               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1288                            locals[i].fib_index, m->proto,
1289                            m - sm->static_mappings);
1290               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1291             }
1292           locals[i].prefix = (i == 0) ? locals[i].probability :
1293             (locals[i - 1].prefix + locals[i].probability);
1294           pool_get (m->locals, local);
1295           *local = locals[i];
1296           if (sm->num_workers > 1)
1297             {
1298               ip4_header_t ip = {
1299                 .src_address = locals[i].addr,
1300               };
1301               bitmap =
1302                 clib_bitmap_set (bitmap,
1303                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1304                                  1);
1305             }
1306         }
1307
1308       /* Assign workers */
1309       if (sm->num_workers > 1)
1310         {
1311           /* *INDENT-OFF* */
1312           clib_bitmap_foreach (i, bitmap)
1313              {
1314                vec_add1(m->workers, i);
1315             }
1316           /* *INDENT-ON* */
1317         }
1318     }
1319   else
1320     {
1321       if (!m)
1322         return VNET_API_ERROR_NO_SUCH_ENTRY;
1323
1324       if (!is_lb_static_mapping (m))
1325         return VNET_API_ERROR_INVALID_VALUE;
1326
1327       /* Free external address port */
1328       if (!(sm->static_mapping_only || out2in_only))
1329         {
1330           for (i = 0; i < vec_len (sm->addresses); i++)
1331             {
1332               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1333                 {
1334                   a = sm->addresses + i;
1335                   switch (proto)
1336                     {
1337 #define _(N, j, n, s) \
1338                     case NAT_PROTOCOL_##N: \
1339                       --a->busy_##n##_port_refcounts[e_port]; \
1340                       if (e_port > 1024) \
1341                         { \
1342                           a->busy_##n##_ports--; \
1343                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1344                         } \
1345                       break;
1346                       foreach_nat_protocol
1347 #undef _
1348                     default:
1349                       nat_elog_info ("unknown protocol");
1350                       return VNET_API_ERROR_INVALID_VALUE_2;
1351                     }
1352                   break;
1353                 }
1354             }
1355         }
1356
1357       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1358       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1359         {
1360           nat_elog_err ("static_mapping_by_external key del failed");
1361           return VNET_API_ERROR_UNSPECIFIED;
1362         }
1363
1364       /* *INDENT-OFF* */
1365       pool_foreach (local, m->locals)
1366       {
1367           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1368                             sm->fib_src_low);
1369           if (!out2in_only)
1370             {
1371 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1372               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1373                 {
1374                   nat_elog_err ("static_mapping_by_local key del failed");
1375                   return VNET_API_ERROR_UNSPECIFIED;
1376                 }
1377             }
1378
1379           if (sm->num_workers > 1)
1380             {
1381               ip4_header_t ip = {
1382                 .src_address = local->addr,
1383               };
1384               tsm = vec_elt_at_index (sm->per_thread_data,
1385                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1386             }
1387           else
1388             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1389
1390           /* Delete sessions */
1391           pool_foreach (s, tsm->sessions) {
1392             if (!(is_lb_session (s)))
1393               continue;
1394
1395             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1396                 s->in2out.port != local->port)
1397               continue;
1398
1399             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1400             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1401           }
1402       }
1403       /* *INDENT-ON* */
1404       if (m->affinity)
1405         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1406       pool_free (m->locals);
1407       vec_free (m->tag);
1408       vec_free (m->workers);
1409
1410       pool_put (sm->static_mappings, m);
1411     }
1412
1413   return 0;
1414 }
1415
1416 int
1417 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1418                                        ip4_address_t l_addr, u16 l_port,
1419                                        nat_protocol_t proto, u32 vrf_id,
1420                                        u8 probability, u8 is_add)
1421 {
1422   snat_main_t *sm = &snat_main;
1423   snat_static_mapping_t *m = 0;
1424   clib_bihash_kv_8_8_t kv, value;
1425   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1426   snat_main_per_thread_data_t *tsm;
1427   snat_session_t *s;
1428   u32 *locals = 0;
1429   uword *bitmap = 0;
1430   int i;
1431
1432   if (!sm->endpoint_dependent)
1433     return VNET_API_ERROR_FEATURE_DISABLED;
1434
1435   init_nat_k (&kv, e_addr, e_port, 0, proto);
1436   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1437     m = pool_elt_at_index (sm->static_mappings, value.value);
1438
1439   if (!m)
1440     return VNET_API_ERROR_NO_SUCH_ENTRY;
1441
1442   if (!is_lb_static_mapping (m))
1443     return VNET_API_ERROR_INVALID_VALUE;
1444
1445   /* *INDENT-OFF* */
1446   pool_foreach (local, m->locals)
1447    {
1448     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1449         (local->vrf_id == vrf_id))
1450       {
1451         match_local = local;
1452         break;
1453       }
1454   }
1455   /* *INDENT-ON* */
1456
1457   if (is_add)
1458     {
1459       if (match_local)
1460         return VNET_API_ERROR_VALUE_EXIST;
1461
1462       pool_get (m->locals, local);
1463       clib_memset (local, 0, sizeof (*local));
1464       local->addr.as_u32 = l_addr.as_u32;
1465       local->port = l_port;
1466       local->probability = probability;
1467       local->vrf_id = vrf_id;
1468       local->fib_index =
1469         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1470                                            sm->fib_src_low);
1471
1472       if (!is_out2in_only_static_mapping (m))
1473         {
1474           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto,
1475                        m - sm->static_mappings);
1476           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1477             nat_elog_err ("static_mapping_by_local key add failed");
1478         }
1479     }
1480   else
1481     {
1482       if (!match_local)
1483         return VNET_API_ERROR_NO_SUCH_ENTRY;
1484
1485       if (pool_elts (m->locals) < 3)
1486         return VNET_API_ERROR_UNSPECIFIED;
1487
1488       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1489                         sm->fib_src_low);
1490
1491       if (!is_out2in_only_static_mapping (m))
1492         {
1493           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1494           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1495             nat_elog_err ("static_mapping_by_local key del failed");
1496         }
1497
1498       if (sm->num_workers > 1)
1499         {
1500           ip4_header_t ip = {
1501             .src_address = local->addr,
1502           };
1503           tsm = vec_elt_at_index (sm->per_thread_data,
1504                                   sm->worker_in2out_cb (&ip, m->fib_index,
1505                                                         0));
1506         }
1507       else
1508         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1509
1510       /* Delete sessions */
1511       /* *INDENT-OFF* */
1512       pool_foreach (s, tsm->sessions) {
1513         if (!(is_lb_session (s)))
1514           continue;
1515
1516         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1517             s->in2out.port != match_local->port)
1518           continue;
1519
1520         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1521         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1522       }
1523       /* *INDENT-ON* */
1524
1525       pool_put (m->locals, match_local);
1526     }
1527
1528   vec_free (m->workers);
1529
1530   /* *INDENT-OFF* */
1531   pool_foreach (local, m->locals)
1532    {
1533     vec_add1 (locals, local - m->locals);
1534     if (sm->num_workers > 1)
1535       {
1536         ip4_header_t ip;
1537         ip.src_address.as_u32 = local->addr.as_u32,
1538         bitmap = clib_bitmap_set (bitmap,
1539                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1540                                   1);
1541       }
1542   }
1543   /* *INDENT-ON* */
1544
1545   ASSERT (vec_len (locals) > 1);
1546
1547   local = pool_elt_at_index (m->locals, locals[0]);
1548   local->prefix = local->probability;
1549   for (i = 1; i < vec_len (locals); i++)
1550     {
1551       local = pool_elt_at_index (m->locals, locals[i]);
1552       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1553       local->prefix = local->probability + prev_local->prefix;
1554     }
1555
1556   /* Assign workers */
1557   if (sm->num_workers > 1)
1558     {
1559       /* *INDENT-OFF* */
1560       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1561       /* *INDENT-ON* */
1562     }
1563
1564   return 0;
1565 }
1566
1567 int
1568 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1569                   u8 twice_nat)
1570 {
1571   snat_address_t *a = 0;
1572   snat_session_t *ses;
1573   u32 *ses_to_be_removed = 0, *ses_index;
1574   snat_main_per_thread_data_t *tsm;
1575   snat_static_mapping_t *m;
1576   snat_interface_t *interface;
1577   int i;
1578   snat_address_t *addresses =
1579     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1580
1581   /* Find SNAT address */
1582   for (i = 0; i < vec_len (addresses); i++)
1583     {
1584       if (addresses[i].addr.as_u32 == addr.as_u32)
1585         {
1586           a = addresses + i;
1587           break;
1588         }
1589     }
1590   if (!a)
1591     {
1592       nat_log_err ("no such address");
1593       return VNET_API_ERROR_NO_SUCH_ENTRY;
1594     }
1595
1596   if (delete_sm)
1597     {
1598       ip4_address_t pool_addr = { 0 };
1599       /* *INDENT-OFF* */
1600       pool_foreach (m, sm->static_mappings)
1601        {
1602           if (m->external_addr.as_u32 == addr.as_u32)
1603             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1604                                             m->local_port, m->external_port,
1605                                             m->vrf_id,
1606                                             is_addr_only_static_mapping(m), ~0,
1607                                             m->proto, 0 /* is_add */,
1608                                             m->twice_nat,
1609                                             is_out2in_only_static_mapping(m),
1610                                             m->tag,
1611                                             is_identity_static_mapping(m),
1612                                             pool_addr, 0);
1613       }
1614       /* *INDENT-ON* */
1615     }
1616   else
1617     {
1618       /* Check if address is used in some static mapping */
1619       if (is_snat_address_used_in_static_mapping (sm, addr))
1620         {
1621           nat_log_err ("address used in static mapping");
1622           return VNET_API_ERROR_UNSPECIFIED;
1623         }
1624     }
1625
1626   if (a->fib_index != ~0)
1627     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1628
1629   /* Delete sessions using address */
1630   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1631     {
1632       vec_foreach (tsm, sm->per_thread_data)
1633       {
1634         /* *INDENT-OFF* */
1635         pool_foreach (ses, tsm->sessions)  {
1636           if (ses->out2in.addr.as_u32 == addr.as_u32)
1637             {
1638               nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1639               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1640             }
1641         }
1642         /* *INDENT-ON* */
1643
1644         if (sm->endpoint_dependent)
1645           {
1646             vec_foreach (ses_index, ses_to_be_removed)
1647             {
1648               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1649               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1650             }
1651           }
1652         else
1653           {
1654             vec_foreach (ses_index, ses_to_be_removed)
1655             {
1656               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1657               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1658             }
1659           }
1660
1661         vec_free (ses_to_be_removed);
1662       }
1663     }
1664
1665 #define _(N, i, n, s) \
1666   vec_free (a->busy_##n##_ports_per_thread);
1667   foreach_nat_protocol
1668 #undef _
1669     if (twice_nat)
1670     {
1671       vec_del1 (sm->twice_nat_addresses, i);
1672       return 0;
1673     }
1674   else
1675     vec_del1 (sm->addresses, i);
1676
1677   /* Delete external address from FIB */
1678   /* *INDENT-OFF* */
1679   pool_foreach (interface, sm->interfaces)
1680    {
1681     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1682       continue;
1683
1684     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1685     break;
1686   }
1687   pool_foreach (interface, sm->output_feature_interfaces)
1688    {
1689     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1690       continue;
1691
1692     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1693     break;
1694   }
1695   /* *INDENT-ON* */
1696
1697   return 0;
1698 }
1699
1700 static void
1701 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1702 {
1703 #define _(x)                                                                  \
1704   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1705                                 sw_if_index);                                 \
1706   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1707   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1708                                 sw_if_index);                                 \
1709   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1710   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1711                                 sw_if_index);                                 \
1712   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1713   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1714                                 sw_if_index);                                 \
1715   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1716   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1717                                 sw_if_index);                                 \
1718   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1719   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1720                                 sw_if_index);                                 \
1721   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1722   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1723                                 sw_if_index);                                 \
1724   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1725   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1726                                 sw_if_index);                                 \
1727   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1728   foreach_nat_counter;
1729 #undef _
1730   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1731   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1732 }
1733
1734 void
1735 expire_per_vrf_sessions (u32 fib_index)
1736 {
1737   per_vrf_sessions_t *per_vrf_sessions;
1738   snat_main_per_thread_data_t *tsm;
1739   snat_main_t *sm = &snat_main;
1740
1741   /* *INDENT-OFF* */
1742   vec_foreach (tsm, sm->per_thread_data)
1743     {
1744       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1745         {
1746           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1747               (per_vrf_sessions->tx_fib_index == fib_index))
1748             {
1749               per_vrf_sessions->expired = 1;
1750             }
1751         }
1752     }
1753   /* *INDENT-ON* */
1754 }
1755
1756 void
1757 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1758 {
1759   snat_main_t *sm = &snat_main;
1760   nat_fib_t *fib;
1761
1762   // we don't care if it is outside/inside fib
1763   // we just care about their ref_count
1764   // if it reaches 0 sessions should expire
1765   // because the fib isn't valid for NAT anymore
1766
1767   vec_foreach (fib, sm->fibs)
1768   {
1769     if (fib->fib_index == fib_index)
1770       {
1771         if (is_del)
1772           {
1773             fib->ref_count--;
1774             if (!fib->ref_count)
1775               {
1776                 vec_del1 (sm->fibs, fib - sm->fibs);
1777                 expire_per_vrf_sessions (fib_index);
1778               }
1779             return;
1780           }
1781         else
1782           fib->ref_count++;
1783       }
1784   }
1785   if (!is_del)
1786     {
1787       vec_add2 (sm->fibs, fib, 1);
1788       fib->ref_count = 1;
1789       fib->fib_index = fib_index;
1790     }
1791 }
1792
1793 int
1794 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1795 {
1796   snat_main_t *sm = &snat_main;
1797   snat_interface_t *i;
1798   const char *feature_name, *del_feature_name;
1799   snat_address_t *ap;
1800   snat_static_mapping_t *m;
1801   nat_outside_fib_t *outside_fib;
1802   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1803                                                        sw_if_index);
1804
1805   if (!sm->enabled)
1806     {
1807       nat_log_err ("nat44 is disabled");
1808       return VNET_API_ERROR_UNSUPPORTED;
1809     }
1810
1811   if (sm->out2in_dpo && !is_inside)
1812     {
1813       nat_log_err ("error unsupported");
1814       return VNET_API_ERROR_UNSUPPORTED;
1815     }
1816
1817   /* *INDENT-OFF* */
1818   pool_foreach (i, sm->output_feature_interfaces)
1819    {
1820     if (i->sw_if_index == sw_if_index)
1821       {
1822         nat_log_err ("error interface already configured");
1823         return VNET_API_ERROR_VALUE_EXIST;
1824       }
1825   }
1826   /* *INDENT-ON* */
1827
1828   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1829     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1830   else
1831     {
1832       if (sm->num_workers > 1)
1833         feature_name =
1834           is_inside ? "nat44-in2out-worker-handoff" :
1835           "nat44-out2in-worker-handoff";
1836       else if (sm->endpoint_dependent)
1837         {
1838           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1839         }
1840       else
1841         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1842     }
1843
1844   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
1845     sm->fq_in2out_index =
1846       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1847
1848   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1849     sm->fq_out2in_index =
1850       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1851
1852   if (sm->endpoint_dependent)
1853     update_per_vrf_sessions_vec (fib_index, is_del);
1854
1855   if (!is_inside)
1856     {
1857       /* *INDENT-OFF* */
1858       vec_foreach (outside_fib, sm->outside_fibs)
1859         {
1860           if (outside_fib->fib_index == fib_index)
1861             {
1862               if (is_del)
1863                 {
1864                   outside_fib->refcount--;
1865                   if (!outside_fib->refcount)
1866                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1867                 }
1868               else
1869                 outside_fib->refcount++;
1870               goto feature_set;
1871             }
1872         }
1873       /* *INDENT-ON* */
1874       if (!is_del)
1875         {
1876           vec_add2 (sm->outside_fibs, outside_fib, 1);
1877           outside_fib->refcount = 1;
1878           outside_fib->fib_index = fib_index;
1879         }
1880     }
1881
1882 feature_set:
1883   /* *INDENT-OFF* */
1884   pool_foreach (i, sm->interfaces)
1885    {
1886     if (i->sw_if_index == sw_if_index)
1887       {
1888         if (is_del)
1889           {
1890             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1891               {
1892                 if (is_inside)
1893                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1894                 else
1895                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1896
1897                 if (sm->num_workers > 1)
1898                   {
1899                     del_feature_name = "nat44-handoff-classify";
1900                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1901                                                  "nat44-out2in-worker-handoff";
1902                   }
1903                 else if (sm->endpoint_dependent)
1904                   {
1905                     del_feature_name = "nat44-ed-classify";
1906                     feature_name = !is_inside ?  "nat-pre-in2out" :
1907                                                  "nat-pre-out2in";
1908                   }
1909                 else
1910                   {
1911                     del_feature_name = "nat44-classify";
1912                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1913                   }
1914
1915                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1916                 if (rv)
1917                   return rv;
1918                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1919                                              sw_if_index, 0, 0, 0);
1920                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1921                                              sw_if_index, 1, 0, 0);
1922                 if (!is_inside)
1923                   {
1924                     if (!sm->endpoint_dependent)
1925                       vnet_feature_enable_disable ("ip4-local",
1926                                                    "nat44-hairpinning",
1927                                                    sw_if_index, 1, 0, 0);
1928                   }
1929               }
1930             else
1931               {
1932                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1933                 if (rv)
1934                   return rv;
1935                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1936                                              sw_if_index, 0, 0, 0);
1937                 pool_put (sm->interfaces, i);
1938                 if (is_inside)
1939                   {
1940                     if (!sm->endpoint_dependent)
1941                       vnet_feature_enable_disable ("ip4-local",
1942                                                    "nat44-hairpinning",
1943                                                    sw_if_index, 0, 0, 0);
1944                   }
1945               }
1946           }
1947         else
1948           {
1949             if ((nat_interface_is_inside (i) && is_inside) ||
1950                 (nat_interface_is_outside (i) && !is_inside))
1951               return 0;
1952
1953             if (sm->num_workers > 1)
1954               {
1955                 del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1956                                                 "nat44-out2in-worker-handoff";
1957                 feature_name = "nat44-handoff-classify";
1958               }
1959             else if (sm->endpoint_dependent)
1960               {
1961                 del_feature_name =
1962                   !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1963
1964                 feature_name = "nat44-ed-classify";
1965               }
1966             else
1967               {
1968                 del_feature_name =
1969                   !is_inside ? "nat44-in2out" : "nat44-out2in";
1970                 feature_name = "nat44-classify";
1971               }
1972
1973             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1974             if (rv)
1975               return rv;
1976             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1977                                          sw_if_index, 0, 0, 0);
1978             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1979                                          sw_if_index, 1, 0, 0);
1980             if (!is_inside)
1981               {
1982                 if (!sm->endpoint_dependent)
1983                   vnet_feature_enable_disable (
1984                     "ip4-local", "nat44-hairpinning", sw_if_index, 0, 0, 0);
1985               }
1986             goto set_flags;
1987           }
1988
1989         goto fib;
1990       }
1991   }
1992   /* *INDENT-ON* */
1993
1994   if (is_del)
1995     {
1996       nat_log_err ("error interface couldn't be found");
1997       return VNET_API_ERROR_NO_SUCH_ENTRY;
1998     }
1999
2000   pool_get (sm->interfaces, i);
2001   i->sw_if_index = sw_if_index;
2002   i->flags = 0;
2003   nat_validate_counters (sm, sw_if_index);
2004
2005   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2006                                0);
2007
2008   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2009   if (rv)
2010     return rv;
2011
2012   if (is_inside && !sm->out2in_dpo)
2013     {
2014       if (!sm->endpoint_dependent)
2015         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2016                                      sw_if_index, 1, 0, 0);
2017     }
2018
2019 set_flags:
2020   if (is_inside)
2021     {
2022       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2023       return 0;
2024     }
2025   else
2026     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2027
2028   /* Add/delete external addresses to FIB */
2029 fib:
2030   /* *INDENT-OFF* */
2031   vec_foreach (ap, sm->addresses)
2032     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2033
2034   pool_foreach (m, sm->static_mappings)
2035    {
2036     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2037       continue;
2038
2039     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2040   }
2041   /* *INDENT-ON* */
2042
2043   return 0;
2044 }
2045
2046 int
2047 snat_interface_add_del_output_feature (u32 sw_if_index,
2048                                        u8 is_inside, int is_del)
2049 {
2050   snat_main_t *sm = &snat_main;
2051   snat_interface_t *i;
2052   snat_address_t *ap;
2053   snat_static_mapping_t *m;
2054   nat_outside_fib_t *outside_fib;
2055   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2056                                                        sw_if_index);
2057
2058   if (!sm->enabled)
2059     {
2060       nat_log_err ("nat44 is disabled");
2061       return VNET_API_ERROR_UNSUPPORTED;
2062     }
2063
2064   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2065     {
2066       nat_log_err ("error unsupported");
2067       return VNET_API_ERROR_UNSUPPORTED;
2068     }
2069
2070   /* *INDENT-OFF* */
2071   pool_foreach (i, sm->interfaces)
2072    {
2073     if (i->sw_if_index == sw_if_index)
2074       {
2075         nat_log_err ("error interface already configured");
2076         return VNET_API_ERROR_VALUE_EXIST;
2077       }
2078   }
2079   /* *INDENT-ON* */
2080
2081   if (sm->endpoint_dependent)
2082     update_per_vrf_sessions_vec (fib_index, is_del);
2083
2084   if (!is_inside)
2085     {
2086       /* *INDENT-OFF* */
2087       vec_foreach (outside_fib, sm->outside_fibs)
2088         {
2089           if (outside_fib->fib_index == fib_index)
2090             {
2091               if (is_del)
2092                 {
2093                   outside_fib->refcount--;
2094                   if (!outside_fib->refcount)
2095                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2096                 }
2097               else
2098                 outside_fib->refcount++;
2099               goto feature_set;
2100             }
2101         }
2102       /* *INDENT-ON* */
2103       if (!is_del)
2104         {
2105           vec_add2 (sm->outside_fibs, outside_fib, 1);
2106           outside_fib->refcount = 1;
2107           outside_fib->fib_index = fib_index;
2108         }
2109     }
2110
2111 feature_set:
2112   if (is_inside)
2113     {
2114       if (sm->endpoint_dependent)
2115         {
2116           int rv =
2117             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2118           if (rv)
2119             return rv;
2120           rv =
2121             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2122                                                             !is_del);
2123           if (rv)
2124             return rv;
2125         }
2126       else
2127         {
2128           int rv =
2129             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2130           if (rv)
2131             return rv;
2132           rv =
2133             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2134                                                             !is_del);
2135           if (rv)
2136             return rv;
2137           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2138                                        sw_if_index, !is_del, 0, 0);
2139           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2140                                        sw_if_index, !is_del, 0, 0);
2141         }
2142       goto fq;
2143     }
2144
2145   if (sm->num_workers > 1)
2146     {
2147       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2148       if (rv)
2149         return rv;
2150       rv =
2151         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2152       if (rv)
2153         return rv;
2154       vnet_feature_enable_disable ("ip4-unicast",
2155                                    "nat44-out2in-worker-handoff",
2156                                    sw_if_index, !is_del, 0, 0);
2157       vnet_feature_enable_disable ("ip4-output",
2158                                    "nat44-in2out-output-worker-handoff",
2159                                    sw_if_index, !is_del, 0, 0);
2160     }
2161   else
2162     {
2163       if (sm->endpoint_dependent)
2164         {
2165           int rv =
2166             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2167           if (rv)
2168             return rv;
2169           rv =
2170             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2171                                                             !is_del);
2172           if (rv)
2173             return rv;
2174           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2175                                        sw_if_index, !is_del, 0, 0);
2176           vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2177                                        sw_if_index, !is_del, 0, 0);
2178         }
2179       else
2180         {
2181           int rv =
2182             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2183           if (rv)
2184             return rv;
2185           rv =
2186             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2187                                                             !is_del);
2188           if (rv)
2189             return rv;
2190           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2191                                        sw_if_index, !is_del, 0, 0);
2192           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2193                                        sw_if_index, !is_del, 0, 0);
2194         }
2195     }
2196
2197 fq:
2198   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2199     sm->fq_in2out_output_index =
2200       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2201
2202   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2203     sm->fq_out2in_index =
2204       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2205
2206   /* *INDENT-OFF* */
2207   pool_foreach (i, sm->output_feature_interfaces)
2208    {
2209     if (i->sw_if_index == sw_if_index)
2210       {
2211         if (is_del)
2212           pool_put (sm->output_feature_interfaces, i);
2213         else
2214           return VNET_API_ERROR_VALUE_EXIST;
2215
2216         goto fib;
2217       }
2218   }
2219   /* *INDENT-ON* */
2220
2221   if (is_del)
2222     {
2223       nat_log_err ("error interface couldn't be found");
2224       return VNET_API_ERROR_NO_SUCH_ENTRY;
2225     }
2226
2227   pool_get (sm->output_feature_interfaces, i);
2228   i->sw_if_index = sw_if_index;
2229   i->flags = 0;
2230   nat_validate_counters (sm, sw_if_index);
2231   if (is_inside)
2232     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2233   else
2234     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2235
2236   /* Add/delete external addresses to FIB */
2237 fib:
2238   if (is_inside)
2239     return 0;
2240
2241   /* *INDENT-OFF* */
2242   vec_foreach (ap, sm->addresses)
2243     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2244
2245   pool_foreach (m, sm->static_mappings)
2246    {
2247     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2248       continue;
2249
2250     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2251   }
2252   /* *INDENT-ON* */
2253
2254   return 0;
2255 }
2256
2257 int
2258 snat_set_workers (uword * bitmap)
2259 {
2260   snat_main_t *sm = &snat_main;
2261   int i, j = 0;
2262
2263   if (sm->num_workers < 2)
2264     return VNET_API_ERROR_FEATURE_DISABLED;
2265
2266   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2267     return VNET_API_ERROR_INVALID_WORKER;
2268
2269   vec_free (sm->workers);
2270   /* *INDENT-OFF* */
2271   clib_bitmap_foreach (i, bitmap)
2272     {
2273       vec_add1(sm->workers, i);
2274       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2275       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2276       j++;
2277     }
2278   /* *INDENT-ON* */
2279
2280   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2281
2282   return 0;
2283 }
2284
2285 static void
2286 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2287                          u32 sw_if_index, u32 new_fib_index,
2288                          u32 old_fib_index)
2289 {
2290   snat_main_t *sm = &snat_main;
2291   nat_outside_fib_t *outside_fib;
2292   snat_interface_t *i;
2293   u8 is_add = 1;
2294   u8 match = 0;
2295
2296   if (!sm->enabled || (new_fib_index == old_fib_index)
2297       || (!vec_len (sm->outside_fibs)))
2298     {
2299       return;
2300     }
2301
2302   /* *INDENT-OFF* */
2303   pool_foreach (i, sm->interfaces)
2304      {
2305       if (i->sw_if_index == sw_if_index)
2306         {
2307           if (!(nat_interface_is_outside (i)))
2308             return;
2309           match = 1;
2310         }
2311     }
2312
2313   pool_foreach (i, sm->output_feature_interfaces)
2314      {
2315       if (i->sw_if_index == sw_if_index)
2316         {
2317           if (!(nat_interface_is_outside (i)))
2318             return;
2319           match = 1;
2320         }
2321     }
2322   /* *INDENT-ON* */
2323
2324   if (!match)
2325     return;
2326
2327   vec_foreach (outside_fib, sm->outside_fibs)
2328   {
2329     if (outside_fib->fib_index == old_fib_index)
2330       {
2331         outside_fib->refcount--;
2332         if (!outside_fib->refcount)
2333           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2334         break;
2335       }
2336   }
2337
2338   vec_foreach (outside_fib, sm->outside_fibs)
2339   {
2340     if (outside_fib->fib_index == new_fib_index)
2341       {
2342         outside_fib->refcount++;
2343         is_add = 0;
2344         break;
2345       }
2346   }
2347
2348   if (is_add)
2349     {
2350       vec_add2 (sm->outside_fibs, outside_fib, 1);
2351       outside_fib->refcount = 1;
2352       outside_fib->fib_index = new_fib_index;
2353     }
2354 }
2355
2356 static void
2357 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2358                          u32 sw_if_index, u32 new_fib_index,
2359                          u32 old_fib_index);
2360
2361 static void
2362 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2363                                        uword opaque,
2364                                        u32 sw_if_index,
2365                                        ip4_address_t * address,
2366                                        u32 address_length,
2367                                        u32 if_address_index, u32 is_delete);
2368
2369 static void
2370 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2371                                  uword opaque,
2372                                  u32 sw_if_index,
2373                                  ip4_address_t * address,
2374                                  u32 address_length,
2375                                  u32 if_address_index, u32 is_delete);
2376
2377 void
2378 test_key_calc_split ()
2379 {
2380   ip4_address_t l_addr;
2381   l_addr.as_u8[0] = 1;
2382   l_addr.as_u8[1] = 1;
2383   l_addr.as_u8[2] = 1;
2384   l_addr.as_u8[3] = 1;
2385   ip4_address_t r_addr;
2386   r_addr.as_u8[0] = 2;
2387   r_addr.as_u8[1] = 2;
2388   r_addr.as_u8[2] = 2;
2389   r_addr.as_u8[3] = 2;
2390   u16 l_port = 40001;
2391   u16 r_port = 40301;
2392   u8 proto = 9;
2393   u32 fib_index = 9000001;
2394   u32 thread_index = 3000000001;
2395   u32 session_index = 3000000221;
2396   clib_bihash_kv_16_8_t kv;
2397   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2398               thread_index, session_index);
2399   ip4_address_t l_addr2;
2400   ip4_address_t r_addr2;
2401   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2402   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2403   u16 l_port2 = 0;
2404   u16 r_port2 = 0;
2405   u8 proto2 = 0;
2406   u32 fib_index2 = 0;
2407   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2408                &r_port2);
2409   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2410   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2411   ASSERT (l_port == l_port2);
2412   ASSERT (r_port == r_port2);
2413   ASSERT (proto == proto2);
2414   ASSERT (fib_index == fib_index2);
2415   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2416   ASSERT (session_index == ed_value_get_session_index (&kv));
2417
2418   fib_index = 7001;
2419   proto = 5;
2420   nat_protocol_t proto3 = ~0;
2421   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2422   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2423   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2424   ASSERT (l_port == l_port2);
2425   ASSERT (proto == proto3);
2426   ASSERT (fib_index == fib_index2);
2427 }
2428
2429 static clib_error_t *
2430 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2431 {
2432   snat_main_t *sm = &snat_main;
2433   u32 fib_index;
2434
2435   if (sm->endpoint_dependent)
2436     {
2437       // TODO: consider removing all NAT interfaces
2438       if (!is_add)
2439         {
2440           fib_index = ip4_fib_index_from_table_id (table_id);
2441           if (fib_index != ~0)
2442             expire_per_vrf_sessions (fib_index);
2443         }
2444     }
2445   return 0;
2446 }
2447
2448 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2449
2450 void
2451 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2452 {
2453   vlib_node_t *node;
2454
2455   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2456   sm->ei_out2in_node_index = node->index;
2457   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2458   sm->ei_in2out_node_index = node->index;
2459   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2460   sm->ei_in2out_output_node_index = node->index;
2461
2462   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2463   sm->ed_out2in_node_index = node->index;
2464   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2465   sm->ed_in2out_node_index = node->index;
2466   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2467   sm->ed_in2out_output_node_index = node->index;
2468
2469   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2470   sm->error_node_index = node->index;
2471   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2472   sm->pre_in2out_node_index = node->index;
2473   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2474   sm->pre_out2in_node_index = node->index;
2475   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2476   sm->pre_in2out_node_index = node->index;
2477   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2478   sm->pre_out2in_node_index = node->index;
2479   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2480   sm->in2out_fast_node_index = node->index;
2481   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2482   sm->in2out_slowpath_node_index = node->index;
2483   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2484   sm->in2out_slowpath_output_node_index = node->index;
2485   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2486   sm->ed_in2out_slowpath_node_index = node->index;
2487   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2488   sm->out2in_fast_node_index = node->index;
2489   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2490   sm->ed_out2in_slowpath_node_index = node->index;
2491   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2492   sm->hairpinning_node_index = node->index;
2493   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2494   sm->hairpin_dst_node_index = node->index;
2495   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2496   sm->hairpin_src_node_index = node->index;
2497 }
2498
2499 #define nat_init_simple_counter(c, n, sn) \
2500 do                                        \
2501   {                                       \
2502     c.name = n;                           \
2503     c.stat_segment_name = sn;             \
2504     vlib_validate_simple_counter (&c, 0); \
2505     vlib_zero_simple_counter (&c, 0);     \
2506   } while (0);
2507
2508 static clib_error_t *
2509 nat_init (vlib_main_t * vm)
2510 {
2511   snat_main_t *sm = &snat_main;
2512   vlib_thread_main_t *tm = vlib_get_thread_main ();
2513   vlib_thread_registration_t *tr;
2514   ip4_add_del_interface_address_callback_t cbi = { 0 };
2515   ip4_table_bind_callback_t cbt = { 0 };
2516   u32 i, num_threads = 0;
2517   uword *p, *bitmap = 0;
2518
2519   clib_memset (sm, 0, sizeof (*sm));
2520
2521   // required
2522   sm->vnet_main = vnet_get_main ();
2523   // convenience
2524   sm->ip4_main = &ip4_main;
2525   sm->api_main = vlibapi_get_main ();
2526   sm->ip4_lookup_main = &ip4_main.lookup_main;
2527
2528   // frame queue indices used for handoff
2529   sm->fq_out2in_index = ~0;
2530   sm->fq_in2out_index = ~0;
2531   sm->fq_in2out_output_index = ~0;
2532
2533   sm->log_level = SNAT_LOG_ERROR;
2534
2535   nat44_set_node_indexes (sm, vm);
2536   sm->log_class = vlib_log_register_class ("nat", 0);
2537   nat_ipfix_logging_init (vm);
2538
2539   nat_init_simple_counter (sm->total_users, "total-users",
2540                            "/nat44/total-users");
2541   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2542                            "/nat44/total-sessions");
2543   nat_init_simple_counter (sm->user_limit_reached, "user-limit-reached",
2544                            "/nat44/user-limit-reached");
2545
2546 #define _(x)                                            \
2547   sm->counters.fastpath.in2out.x.name = #x;             \
2548   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2549       "/nat44/in2out/fastpath/" #x;                     \
2550   sm->counters.slowpath.in2out.x.name = #x;             \
2551   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2552       "/nat44/in2out/slowpath/" #x;                     \
2553   sm->counters.fastpath.out2in.x.name = #x;             \
2554   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2555       "/nat44/out2in/fastpath/" #x;                     \
2556   sm->counters.slowpath.out2in.x.name = #x;             \
2557   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2558       "/nat44/out2in/slowpath/" #x;                     \
2559   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2560   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2561       "/nat44/ed/in2out/fastpath/" #x;                  \
2562   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2563   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2564       "/nat44/ed/in2out/slowpath/" #x;                  \
2565   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2566   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2567       "/nat44/ed/out2in/fastpath/" #x;                  \
2568   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2569   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2570       "/nat44/ed/out2in/slowpath/" #x;
2571   foreach_nat_counter;
2572 #undef _
2573   sm->counters.hairpinning.name = "hairpinning";
2574   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2575
2576   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2577   if (p)
2578     {
2579       tr = (vlib_thread_registration_t *) p[0];
2580       if (tr)
2581         {
2582           sm->num_workers = tr->count;
2583           sm->first_worker_index = tr->first_index;
2584         }
2585     }
2586   num_threads = tm->n_vlib_mains - 1;
2587   sm->port_per_thread = 0xffff - 1024;
2588   vec_validate (sm->per_thread_data, num_threads);
2589
2590   /* Use all available workers by default */
2591   if (sm->num_workers > 1)
2592     {
2593
2594       for (i = 0; i < sm->num_workers; i++)
2595         bitmap = clib_bitmap_set (bitmap, i, 1);
2596       snat_set_workers (bitmap);
2597       clib_bitmap_free (bitmap);
2598     }
2599   else
2600     sm->per_thread_data[0].snat_thread_index = 0;
2601
2602   /* callbacks to call when interface address changes. */
2603   cbi.function = snat_ip4_add_del_interface_address_cb;
2604   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2605   cbi.function = nat_ip4_add_del_addr_only_sm_cb;
2606   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2607
2608   /* callbacks to call when interface to table biding changes */
2609   cbt.function = snat_update_outside_fib;
2610   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2611
2612   sm->fib_src_low =
2613     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2614                          FIB_SOURCE_BH_SIMPLE);
2615   sm->fib_src_hi =
2616     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2617                          FIB_SOURCE_BH_SIMPLE);
2618
2619   /* used only by out2in-dpo feature */
2620   nat_dpo_module_init ();
2621
2622   nat_affinity_init (vm);
2623   nat_ha_init (vm, sm->num_workers, num_threads);
2624
2625   test_key_calc_split ();
2626   return nat44_api_hookup (vm);
2627 }
2628
2629 VLIB_INIT_FUNCTION (nat_init);
2630
2631 static int
2632 nat44_ed_plugin_enable (nat44_config_t c)
2633 {
2634   snat_main_t *sm = &snat_main;
2635
2636   if (c.static_mapping_only && !c.connection_tracking)
2637     {
2638       nat_log_err ("unsupported combination of configuration");
2639       return 1;
2640     }
2641
2642   // nat44 feature configuration
2643   sm->endpoint_dependent = c.endpoint_dependent;
2644   sm->static_mapping_only = c.static_mapping_only;
2645   sm->static_mapping_connection_tracking = c.connection_tracking;
2646
2647   // EI only feature (could break ED)
2648   sm->out2in_dpo = c.out2in_dpo;
2649
2650   sm->forwarding_enabled = 0;
2651   sm->mss_clamping = 0;
2652   sm->pat = (!c.static_mapping_only ||
2653              (c.static_mapping_only && c.connection_tracking));
2654
2655   if (!c.users)
2656     c.users = 1024;
2657
2658   // EI only feature (could break ED)
2659   sm->max_users_per_thread = c.users;
2660   sm->user_buckets = nat_calc_bihash_buckets (c.users);
2661
2662   if (!c.sessions)
2663     c.sessions = 10 * 1024;
2664
2665   sm->max_translations_per_thread = c.sessions;
2666   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2667
2668   // ED only feature
2669   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2670
2671   // EI only feature (could break ED)
2672   sm->max_translations_per_user
2673     = c.user_sessions ? c.user_sessions : sm->max_translations_per_thread;
2674
2675   sm->inside_vrf_id = c.inside_vrf;
2676   sm->inside_fib_index =
2677     fib_table_find_or_create_and_lock
2678     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2679
2680   sm->outside_vrf_id = c.outside_vrf;
2681   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2682     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2683
2684   sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
2685   sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2686
2687   sm->in2out_node_index = sm->ed_in2out_node_index;
2688   sm->out2in_node_index = sm->ed_out2in_node_index;
2689
2690   sm->in2out_output_node_index = sm->ed_in2out_output_node_index;
2691
2692   if (sm->pat)
2693     {
2694       sm->icmp_match_in2out_cb = NULL;
2695       sm->icmp_match_out2in_cb = NULL;
2696     }
2697   else
2698     {
2699       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2700       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2701     }
2702
2703   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets,
2704                     sm->user_buckets);
2705
2706   nat_affinity_enable ();
2707
2708   nat_reset_timeouts (&sm->timeouts);
2709
2710   // TODO: function for reset counters
2711   vlib_zero_simple_counter (&sm->total_users, 0);
2712   vlib_zero_simple_counter (&sm->total_sessions, 0);
2713   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2714
2715   sm->enabled = 1;
2716   sm->rconfig = c;
2717
2718   return 0;
2719 }
2720
2721 int
2722 nat44_plugin_enable (nat44_config_t c)
2723 {
2724   fail_if_enabled ();
2725
2726   // c.static_mapping_only + c.connection_tracking
2727   //  - supported in NAT EI & NAT ED
2728   // c.out2in_dpo, c.static_mapping_only
2729   //  - supported in NAT EI
2730
2731   if (c.endpoint_dependent)
2732     {
2733       if (c.out2in_dpo || c.users || c.user_sessions)
2734         {
2735           nat_log_err ("unsupported combination of configuration");
2736           return 1;
2737         }
2738       return nat44_ed_plugin_enable (c);
2739     }
2740
2741   // separation:
2742   // for now just copy variables
2743
2744   nat44_ei_config_t ei_c = {
2745     .inside_vrf = c.inside_vrf,
2746     .outside_vrf = c.outside_vrf,
2747     .users = c.users,
2748     .sessions = c.sessions,
2749     .user_sessions = c.user_sessions,
2750     .out2in_dpo = c.out2in_dpo,
2751     .static_mapping_only = c.static_mapping_only,
2752     .connection_tracking = c.connection_tracking,
2753   };
2754
2755   return nat44_ei_plugin_enable (ei_c);
2756 }
2757
2758 void
2759 nat44_addresses_free (snat_address_t ** addresses)
2760 {
2761   snat_address_t *ap;
2762   /* *INDENT-OFF* */
2763   vec_foreach (ap, *addresses)
2764     {
2765     #define _(N, i, n, s) \
2766       vec_free (ap->busy_##n##_ports_per_thread);
2767       foreach_nat_protocol
2768     #undef _
2769     }
2770   /* *INDENT-ON* */
2771   vec_free (*addresses);
2772   *addresses = 0;
2773 }
2774
2775 static int
2776 nat44_ed_plugin_disable ()
2777 {
2778   snat_main_t *sm = &snat_main;
2779   snat_interface_t *i, *vec;
2780   int error = 0;
2781
2782   // first unregister all nodes from interfaces
2783   vec = vec_dup (sm->interfaces);
2784   /* *INDENT-OFF* */
2785   vec_foreach (i, vec)
2786     {
2787       if (nat_interface_is_inside(i))
2788         error = snat_interface_add_del (i->sw_if_index, 1, 1);
2789       if (nat_interface_is_outside(i))
2790         error = snat_interface_add_del (i->sw_if_index, 0, 1);
2791
2792       if (error)
2793         {
2794           nat_log_err ("error occurred while removing interface %u",
2795                        i->sw_if_index);
2796         }
2797     }
2798   /* *INDENT-ON* */
2799   vec_free (vec);
2800   sm->interfaces = 0;
2801
2802   vec = vec_dup (sm->output_feature_interfaces);
2803   /* *INDENT-OFF* */
2804   vec_foreach (i, vec)
2805     {
2806       if (nat_interface_is_inside(i))
2807         error = snat_interface_add_del_output_feature (i->sw_if_index, 1, 1);
2808       if (nat_interface_is_outside(i))
2809         error = snat_interface_add_del_output_feature (i->sw_if_index, 0, 1);
2810
2811       if (error)
2812         {
2813           nat_log_err ("error occurred while removing interface %u",
2814                        i->sw_if_index);
2815         }
2816     }
2817   /* *INDENT-ON* */
2818   vec_free (vec);
2819   sm->output_feature_interfaces = 0;
2820
2821   vec_free (sm->max_translations_per_fib);
2822
2823   nat44_ed_db_free ();
2824
2825   nat44_addresses_free (&sm->addresses);
2826   nat44_addresses_free (&sm->twice_nat_addresses);
2827
2828
2829   vec_free (sm->to_resolve);
2830   vec_free (sm->auto_add_sw_if_indices);
2831   vec_free (sm->auto_add_sw_if_indices_twice_nat);
2832
2833   sm->to_resolve = 0;
2834   sm->auto_add_sw_if_indices = 0;
2835   sm->auto_add_sw_if_indices_twice_nat = 0;
2836
2837   sm->forwarding_enabled = 0;
2838
2839   sm->enabled = 0;
2840   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2841
2842   return 0;
2843 }
2844
2845 int
2846 nat44_plugin_disable ()
2847 {
2848   snat_main_t *sm = &snat_main;
2849
2850   fail_if_disabled ();
2851
2852   if (sm->endpoint_dependent)
2853     return nat44_ed_plugin_disable ();
2854   return nat44_ei_plugin_disable ();
2855 }
2856
2857 void
2858 snat_free_outside_address_and_port (snat_address_t *addresses,
2859                                     u32 thread_index, ip4_address_t *addr,
2860                                     u16 port, nat_protocol_t protocol)
2861 {
2862   snat_address_t *a;
2863   u32 address_index;
2864   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2865
2866   for (address_index = 0; address_index < vec_len (addresses);
2867        address_index++)
2868     {
2869       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2870         break;
2871     }
2872
2873   ASSERT (address_index < vec_len (addresses));
2874
2875   a = addresses + address_index;
2876
2877   switch (protocol)
2878     {
2879 #define _(N, i, n, s) \
2880     case NAT_PROTOCOL_##N: \
2881       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2882       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2883       a->busy_##n##_ports--; \
2884       a->busy_##n##_ports_per_thread[thread_index]--; \
2885       break;
2886       foreach_nat_protocol
2887 #undef _
2888     default:
2889       nat_elog_info ("unknown protocol");
2890       return;
2891     }
2892 }
2893
2894 int
2895 nat_set_outside_address_and_port (snat_address_t *addresses, u32 thread_index,
2896                                   ip4_address_t addr, u16 port,
2897                                   nat_protocol_t protocol)
2898 {
2899   snat_address_t *a = 0;
2900   u32 address_index;
2901   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2902
2903   for (address_index = 0; address_index < vec_len (addresses);
2904        address_index++)
2905     {
2906       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2907         continue;
2908
2909       a = addresses + address_index;
2910       switch (protocol)
2911         {
2912 #define _(N, j, n, s) \
2913         case NAT_PROTOCOL_##N: \
2914           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2915             return VNET_API_ERROR_INSTANCE_IN_USE; \
2916           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2917           a->busy_##n##_ports_per_thread[thread_index]++; \
2918           a->busy_##n##_ports++; \
2919           return 0;
2920           foreach_nat_protocol
2921 #undef _
2922         default:
2923           nat_elog_info ("unknown protocol");
2924           return 1;
2925         }
2926     }
2927
2928   return VNET_API_ERROR_NO_SUCH_ENTRY;
2929 }
2930
2931 int
2932 snat_static_mapping_match (snat_main_t * sm,
2933                            ip4_address_t match_addr,
2934                            u16 match_port,
2935                            u32 match_fib_index,
2936                            nat_protocol_t match_protocol,
2937                            ip4_address_t * mapping_addr,
2938                            u16 * mapping_port,
2939                            u32 * mapping_fib_index,
2940                            u8 by_external,
2941                            u8 * is_addr_only,
2942                            twice_nat_type_t * twice_nat,
2943                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2944                            u8 * is_identity_nat, snat_static_mapping_t ** out)
2945 {
2946   clib_bihash_kv_8_8_t kv, value;
2947   clib_bihash_8_8_t *mapping_hash;
2948   snat_static_mapping_t *m;
2949   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2950   nat44_lb_addr_port_t *local;
2951   u8 backend_index;
2952
2953   if (!by_external)
2954     {
2955       mapping_hash = &sm->static_mapping_by_local;
2956       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2957                   match_protocol);
2958       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2959         {
2960           /* Try address only mapping */
2961           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2962           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2963             return 1;
2964         }
2965     }
2966   else
2967     {
2968       mapping_hash = &sm->static_mapping_by_external;
2969       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2970       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2971         {
2972           /* Try address only mapping */
2973           init_nat_k (&kv, match_addr, 0, 0, 0);
2974           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2975             return 1;
2976         }
2977     }
2978
2979   m = pool_elt_at_index (sm->static_mappings, value.value);
2980
2981   if (by_external)
2982     {
2983       if (is_lb_static_mapping (m))
2984         {
2985           if (PREDICT_FALSE (lb != 0))
2986             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2987           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2988                                                           match_addr,
2989                                                           match_protocol,
2990                                                           match_port,
2991                                                           &backend_index))
2992             {
2993               local = pool_elt_at_index (m->locals, backend_index);
2994               *mapping_addr = local->addr;
2995               *mapping_port = local->port;
2996               *mapping_fib_index = local->fib_index;
2997               goto end;
2998             }
2999           // pick locals matching this worker
3000           if (PREDICT_FALSE (sm->num_workers > 1))
3001             {
3002               u32 thread_index = vlib_get_thread_index ();
3003               /* *INDENT-OFF* */
3004               pool_foreach_index (i, m->locals)
3005                {
3006                 local = pool_elt_at_index (m->locals, i);
3007
3008                 ip4_header_t ip = {
3009                   .src_address = local->addr,
3010                 };
3011
3012                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
3013                     thread_index)
3014                   {
3015                     vec_add1 (tmp, i);
3016                   }
3017               }
3018               /* *INDENT-ON* */
3019               ASSERT (vec_len (tmp) != 0);
3020             }
3021           else
3022             {
3023               /* *INDENT-OFF* */
3024               pool_foreach_index (i, m->locals)
3025                {
3026                 vec_add1 (tmp, i);
3027               }
3028               /* *INDENT-ON* */
3029             }
3030           hi = vec_len (tmp) - 1;
3031           local = pool_elt_at_index (m->locals, tmp[hi]);
3032           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
3033           while (lo < hi)
3034             {
3035               mid = ((hi - lo) >> 1) + lo;
3036               local = pool_elt_at_index (m->locals, tmp[mid]);
3037               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
3038             }
3039           local = pool_elt_at_index (m->locals, tmp[lo]);
3040           if (!(local->prefix >= rand))
3041             return 1;
3042           *mapping_addr = local->addr;
3043           *mapping_port = local->port;
3044           *mapping_fib_index = local->fib_index;
3045           if (m->affinity)
3046             {
3047               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
3048                                                 match_protocol, match_port,
3049                                                 tmp[lo], m->affinity,
3050                                                 m->affinity_per_service_list_head_index))
3051                 nat_elog_info ("create affinity record failed");
3052             }
3053           vec_free (tmp);
3054         }
3055       else
3056         {
3057           if (PREDICT_FALSE (lb != 0))
3058             *lb = NO_LB_NAT;
3059           *mapping_fib_index = m->fib_index;
3060           *mapping_addr = m->local_addr;
3061           /* Address only mapping doesn't change port */
3062           *mapping_port = is_addr_only_static_mapping (m) ? match_port
3063             : m->local_port;
3064         }
3065     }
3066   else
3067     {
3068       *mapping_addr = m->external_addr;
3069       /* Address only mapping doesn't change port */
3070       *mapping_port = is_addr_only_static_mapping (m) ? match_port
3071         : m->external_port;
3072       *mapping_fib_index = sm->outside_fib_index;
3073     }
3074
3075 end:
3076   if (PREDICT_FALSE (is_addr_only != 0))
3077     *is_addr_only = is_addr_only_static_mapping (m);
3078
3079   if (PREDICT_FALSE (twice_nat != 0))
3080     *twice_nat = m->twice_nat;
3081
3082   if (PREDICT_FALSE (is_identity_nat != 0))
3083     *is_identity_nat = is_identity_static_mapping (m);
3084
3085   if (out != 0)
3086     *out = m;
3087
3088   return 0;
3089 }
3090
3091 void
3092 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3093 {
3094   snat_main_t *sm = &snat_main;
3095   dpo_id_t dpo_v4 = DPO_INVALID;
3096   fib_prefix_t pfx = {
3097     .fp_proto = FIB_PROTOCOL_IP4,
3098     .fp_len = 32,
3099     .fp_addr.ip4.as_u32 = addr.as_u32,
3100   };
3101
3102   if (is_add)
3103     {
3104       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3105       fib_table_entry_special_dpo_add (0, &pfx, sm->fib_src_hi,
3106                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3107       dpo_reset (&dpo_v4);
3108     }
3109   else
3110     {
3111       fib_table_entry_special_remove (0, &pfx, sm->fib_src_hi);
3112     }
3113 }
3114
3115 static u32
3116 nat44_ed_get_worker_in2out_cb (ip4_header_t *ip, u32 rx_fib_index,
3117                                u8 is_output)
3118 {
3119   snat_main_t *sm = &snat_main;
3120   u32 next_worker_index = sm->first_worker_index;
3121   u32 hash;
3122
3123   clib_bihash_kv_16_8_t kv16, value16;
3124   snat_main_per_thread_data_t *tsm;
3125   udp_header_t *udp;
3126
3127   if (PREDICT_FALSE (is_output))
3128     {
3129       u32 fib_index = sm->outside_fib_index;
3130       nat_outside_fib_t *outside_fib;
3131       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3132       fib_prefix_t pfx = {
3133         .fp_proto = FIB_PROTOCOL_IP4,
3134         .fp_len = 32,
3135         .fp_addr = {
3136                     .ip4.as_u32 = ip->dst_address.as_u32,
3137                     }
3138         ,
3139       };
3140
3141       udp = ip4_next_header (ip);
3142
3143       switch (vec_len (sm->outside_fibs))
3144         {
3145         case 0:
3146           fib_index = sm->outside_fib_index;
3147           break;
3148         case 1:
3149           fib_index = sm->outside_fibs[0].fib_index;
3150           break;
3151         default:
3152             /* *INDENT-OFF* */
3153             vec_foreach (outside_fib, sm->outside_fibs)
3154               {
3155                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3156                 if (FIB_NODE_INDEX_INVALID != fei)
3157                   {
3158                     if (fib_entry_get_resolving_interface (fei) != ~0)
3159                       {
3160                         fib_index = outside_fib->fib_index;
3161                         break;
3162                       }
3163                   }
3164               }
3165             /* *INDENT-ON* */
3166           break;
3167         }
3168
3169       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3170                  udp->dst_port, fib_index, ip->protocol);
3171
3172       if (PREDICT_TRUE (
3173             !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3174         {
3175           tsm =
3176             vec_elt_at_index (sm->per_thread_data,
3177                               ed_value_get_thread_index (&value16));
3178           next_worker_index += tsm->thread_index;
3179
3180           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3181                                   next_worker_index, fib_index,
3182                                   clib_net_to_host_u32 (ip->
3183                                                         src_address.as_u32),
3184                                   clib_net_to_host_u32 (ip->
3185                                                         dst_address.as_u32));
3186
3187           return next_worker_index;
3188         }
3189     }
3190
3191   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3192     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3193
3194   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3195     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3196   else
3197     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3198
3199   if (PREDICT_TRUE (!is_output))
3200     {
3201       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3202                               next_worker_index, rx_fib_index,
3203                               clib_net_to_host_u32 (ip->src_address.as_u32),
3204                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3205     }
3206   else
3207     {
3208       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3209                               next_worker_index, rx_fib_index,
3210                               clib_net_to_host_u32 (ip->src_address.as_u32),
3211                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3212     }
3213
3214   return next_worker_index;
3215 }
3216
3217 static u32
3218 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3219                                u32 rx_fib_index, u8 is_output)
3220 {
3221   snat_main_t *sm = &snat_main;
3222   clib_bihash_kv_8_8_t kv, value;
3223   clib_bihash_kv_16_8_t kv16, value16;
3224   snat_main_per_thread_data_t *tsm;
3225
3226   u32 proto, next_worker_index = 0;
3227   udp_header_t *udp;
3228   u16 port;
3229   snat_static_mapping_t *m;
3230   u32 hash;
3231
3232   proto = ip_proto_to_nat_proto (ip->protocol);
3233
3234   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3235     {
3236       udp = ip4_next_header (ip);
3237
3238       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3239                  udp->src_port, rx_fib_index, ip->protocol);
3240
3241       if (PREDICT_TRUE (
3242             !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3243         {
3244           tsm =
3245             vec_elt_at_index (sm->per_thread_data,
3246                               ed_value_get_thread_index (&value16));
3247           vnet_buffer2 (b)->nat.cached_session_index =
3248             ed_value_get_session_index (&value16);
3249           next_worker_index = sm->first_worker_index + tsm->thread_index;
3250           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3251                                   next_worker_index, rx_fib_index,
3252                                   clib_net_to_host_u32 (ip->
3253                                                         src_address.as_u32),
3254                                   clib_net_to_host_u32 (ip->
3255                                                         dst_address.as_u32));
3256           return next_worker_index;
3257         }
3258     }
3259   else if (proto == NAT_PROTOCOL_ICMP)
3260     {
3261       ip4_address_t lookup_saddr, lookup_daddr;
3262       u16 lookup_sport, lookup_dport;
3263       u8 lookup_protocol;
3264       if (!nat_get_icmp_session_lookup_values (
3265             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3266             &lookup_protocol))
3267         {
3268           init_ed_k (&kv16, lookup_saddr, lookup_sport, lookup_daddr,
3269                      lookup_dport, rx_fib_index, lookup_protocol);
3270           if (PREDICT_TRUE (
3271                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3272             {
3273               tsm =
3274                 vec_elt_at_index (sm->per_thread_data,
3275                                   ed_value_get_thread_index (&value16));
3276               next_worker_index = sm->first_worker_index + tsm->thread_index;
3277               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3278                                       next_worker_index, rx_fib_index,
3279                                       clib_net_to_host_u32 (ip->
3280                                                             src_address.as_u32),
3281                                       clib_net_to_host_u32 (ip->
3282                                                             dst_address.as_u32));
3283               return next_worker_index;
3284             }
3285         }
3286     }
3287
3288   /* first try static mappings without port */
3289   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3290     {
3291       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3292       if (!clib_bihash_search_8_8
3293           (&sm->static_mapping_by_external, &kv, &value))
3294         {
3295           m = pool_elt_at_index (sm->static_mappings, value.value);
3296           next_worker_index = m->workers[0];
3297           goto done;
3298         }
3299     }
3300
3301   /* unknown protocol */
3302   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3303     {
3304       /* use current thread */
3305       next_worker_index = vlib_get_thread_index ();
3306       goto done;
3307     }
3308
3309   udp = ip4_next_header (ip);
3310   port = udp->dst_port;
3311
3312   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3313     {
3314       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3315       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3316       if (!icmp_type_is_error_message
3317           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3318         port = vnet_buffer (b)->ip.reass.l4_src_port;
3319       else
3320         {
3321           /* if error message, then it's not fragmented and we can access it */
3322           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3323           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3324           void *l4_header = ip4_next_header (inner_ip);
3325           switch (proto)
3326             {
3327             case NAT_PROTOCOL_ICMP:
3328               icmp = (icmp46_header_t *) l4_header;
3329               echo = (icmp_echo_header_t *) (icmp + 1);
3330               port = echo->identifier;
3331               break;
3332             case NAT_PROTOCOL_UDP:
3333             case NAT_PROTOCOL_TCP:
3334               port = ((tcp_udp_header_t *) l4_header)->src_port;
3335               break;
3336             default:
3337               next_worker_index = vlib_get_thread_index ();
3338               goto done;
3339             }
3340         }
3341     }
3342
3343   /* try static mappings with port */
3344   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3345     {
3346       init_nat_k (&kv, ip->dst_address, port, 0, proto);
3347       if (!clib_bihash_search_8_8
3348           (&sm->static_mapping_by_external, &kv, &value))
3349         {
3350           m = pool_elt_at_index (sm->static_mappings, value.value);
3351           if (!is_lb_static_mapping (m))
3352             {
3353               next_worker_index = m->workers[0];
3354               goto done;
3355             }
3356
3357           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3358             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3359
3360           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3361             next_worker_index =
3362               m->workers[hash & (_vec_len (m->workers) - 1)];
3363           else
3364             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3365           goto done;
3366         }
3367     }
3368
3369   /* worker by outside port */
3370   next_worker_index = sm->first_worker_index;
3371   next_worker_index +=
3372     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3373
3374 done:
3375   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3376                           clib_net_to_host_u32 (ip->src_address.as_u32),
3377                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3378   return next_worker_index;
3379 }
3380
3381 u32
3382 nat_calc_bihash_buckets (u32 n_elts)
3383 {
3384   n_elts = n_elts / 2.5;
3385   u64 lower_pow2 = 1;
3386   while (lower_pow2 * 2 < n_elts)
3387     {
3388       lower_pow2 = 2 * lower_pow2;
3389     }
3390   u64 upper_pow2 = 2 * lower_pow2;
3391   if ((upper_pow2 - n_elts) < (n_elts - lower_pow2))
3392     {
3393       if (upper_pow2 <= UINT32_MAX)
3394         {
3395           return upper_pow2;
3396         }
3397     }
3398   return lower_pow2;
3399 }
3400
3401 u32
3402 nat44_get_max_session_limit ()
3403 {
3404   snat_main_t *sm = &snat_main;
3405   u32 max_limit = 0, len = 0;
3406
3407   for (; len < vec_len (sm->max_translations_per_fib); len++)
3408     {
3409       if (max_limit < sm->max_translations_per_fib[len])
3410         max_limit = sm->max_translations_per_fib[len];
3411     }
3412   return max_limit;
3413 }
3414
3415 int
3416 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3417 {
3418   snat_main_t *sm = &snat_main;
3419   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3420   u32 len = vec_len (sm->max_translations_per_fib);
3421
3422   if (len <= fib_index)
3423     {
3424       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3425
3426       for (; len < vec_len (sm->max_translations_per_fib); len++)
3427         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3428     }
3429
3430   sm->max_translations_per_fib[fib_index] = session_limit;
3431   return 0;
3432 }
3433
3434 int
3435 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3436 {
3437   snat_main_t *sm = &snat_main;
3438
3439   if (nat44_set_session_limit (session_limit, vrf_id))
3440     return 1;
3441   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3442
3443   sm->translation_buckets =
3444     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3445
3446   nat44_sessions_clear ();
3447   return 0;
3448 }
3449
3450 static void
3451 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
3452                          u32 translation_buckets, u32 user_buckets)
3453 {
3454   dlist_elt_t *head;
3455
3456   pool_alloc (tsm->sessions, translations);
3457   pool_alloc (tsm->lru_pool, translations);
3458
3459   pool_get (tsm->lru_pool, head);
3460   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3461   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3462
3463   pool_get (tsm->lru_pool, head);
3464   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3465   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3466
3467   pool_get (tsm->lru_pool, head);
3468   tsm->udp_lru_head_index = head - tsm->lru_pool;
3469   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3470
3471   pool_get (tsm->lru_pool, head);
3472   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3473   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3474
3475   pool_get (tsm->lru_pool, head);
3476   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3477   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3478
3479   // TODO: ED nat is not using these
3480   // before removal large refactor required
3481   pool_alloc (tsm->list_pool, translations);
3482   clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets, 0);
3483   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3484 }
3485
3486 static void
3487 reinit_ed_flow_hash ()
3488 {
3489   snat_main_t *sm = &snat_main;
3490   // we expect 2 flows per session, so multiply translation_buckets by 2
3491   clib_bihash_init_16_8 (
3492     &sm->flow_hash, "ed-flow-hash",
3493     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3494   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3495 }
3496
3497 static void
3498 nat44_ed_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
3499 {
3500   snat_main_t *sm = &snat_main;
3501   snat_main_per_thread_data_t *tsm;
3502   u32 static_mapping_buckets = 1024;
3503   u32 static_mapping_memory_size = 64 << 20;
3504
3505   reinit_ed_flow_hash ();
3506
3507   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3508                         "static_mapping_by_local", static_mapping_buckets,
3509                         static_mapping_memory_size);
3510   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3511                                      format_static_mapping_kvp);
3512
3513   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3514                         "static_mapping_by_external", static_mapping_buckets,
3515                         static_mapping_memory_size);
3516   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3517                                      format_static_mapping_kvp);
3518
3519   if (sm->pat)
3520     {
3521       vec_foreach (tsm, sm->per_thread_data)
3522         {
3523           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3524                                    sm->translation_buckets, sm->user_buckets);
3525         }
3526     }
3527 }
3528
3529 static void
3530 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3531 {
3532   pool_free (tsm->sessions);
3533   pool_free (tsm->lru_pool);
3534
3535   vec_free (tsm->per_vrf_sessions_vec);
3536
3537   // TODO: resolve static mappings (put only to !ED)
3538   pool_free (tsm->list_pool);
3539   pool_free (tsm->users);
3540   clib_bihash_free_8_8 (&tsm->user_hash);
3541 }
3542
3543 static void
3544 nat44_ed_db_free ()
3545 {
3546   snat_main_t *sm = &snat_main;
3547   snat_main_per_thread_data_t *tsm;
3548
3549   pool_free (sm->static_mappings);
3550   clib_bihash_free_16_8 (&sm->flow_hash);
3551   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
3552   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
3553
3554   if (sm->pat)
3555     {
3556       vec_foreach (tsm, sm->per_thread_data)
3557         {
3558           nat44_ed_worker_db_free (tsm);
3559         }
3560     }
3561 }
3562
3563 void
3564 nat44_ed_sessions_clear ()
3565 {
3566   snat_main_t *sm = &snat_main;
3567   snat_main_per_thread_data_t *tsm;
3568
3569   reinit_ed_flow_hash ();
3570
3571   if (sm->pat)
3572     {
3573       vec_foreach (tsm, sm->per_thread_data)
3574         {
3575
3576           nat44_ed_worker_db_free (tsm);
3577           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3578                                    sm->translation_buckets, sm->user_buckets);
3579         }
3580     }
3581
3582   // TODO: function for reset counters
3583   vlib_zero_simple_counter (&sm->total_users, 0);
3584   vlib_zero_simple_counter (&sm->total_sessions, 0);
3585   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
3586 }
3587
3588 void
3589 nat44_sessions_clear ()
3590 {
3591   snat_main_t *sm = &snat_main;
3592
3593   if (sm->endpoint_dependent)
3594     nat44_ed_sessions_clear ();
3595   else
3596     nat44_ei_sessions_clear ();
3597 }
3598
3599 static void
3600 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3601                                  uword opaque,
3602                                  u32 sw_if_index,
3603                                  ip4_address_t * address,
3604                                  u32 address_length,
3605                                  u32 if_address_index, u32 is_delete)
3606 {
3607   snat_main_t *sm = &snat_main;
3608   snat_static_map_resolve_t *rp;
3609   snat_static_mapping_t *m;
3610   clib_bihash_kv_8_8_t kv, value;
3611   int i, rv;
3612   ip4_address_t l_addr;
3613
3614   if (!sm->enabled)
3615     return;
3616
3617   for (i = 0; i < vec_len (sm->to_resolve); i++)
3618     {
3619       rp = sm->to_resolve + i;
3620       if (rp->addr_only == 0)
3621         continue;
3622       if (rp->sw_if_index == sw_if_index)
3623         goto match;
3624     }
3625
3626   return;
3627
3628 match:
3629   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
3630               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
3631   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3632     m = 0;
3633   else
3634     m = pool_elt_at_index (sm->static_mappings, value.value);
3635
3636   if (!is_delete)
3637     {
3638       /* Don't trip over lease renewal, static config */
3639       if (m)
3640         return;
3641     }
3642   else
3643     {
3644       if (!m)
3645         return;
3646     }
3647
3648   /* Indetity mapping? */
3649   if (rp->l_addr.as_u32 == 0)
3650     l_addr.as_u32 = address[0].as_u32;
3651   else
3652     l_addr.as_u32 = rp->l_addr.as_u32;
3653   /* Add the static mapping */
3654   rv = snat_add_static_mapping (l_addr,
3655                                 address[0],
3656                                 rp->l_port,
3657                                 rp->e_port,
3658                                 rp->vrf_id,
3659                                 rp->addr_only, ~0 /* sw_if_index */ ,
3660                                 rp->proto, !is_delete, rp->twice_nat,
3661                                 rp->out2in_only, rp->tag, rp->identity_nat,
3662                                 rp->pool_addr, rp->exact);
3663   if (rv)
3664     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
3665 }
3666
3667 static void
3668 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3669                                        uword opaque,
3670                                        u32 sw_if_index,
3671                                        ip4_address_t * address,
3672                                        u32 address_length,
3673                                        u32 if_address_index, u32 is_delete)
3674 {
3675   snat_main_t *sm = &snat_main;
3676   snat_static_map_resolve_t *rp;
3677   ip4_address_t l_addr;
3678   int i, j;
3679   int rv;
3680   u8 twice_nat = 0;
3681   snat_address_t *addresses = sm->addresses;
3682
3683   if (!sm->enabled)
3684     return;
3685
3686   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3687     {
3688       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3689         goto match;
3690     }
3691
3692   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3693     {
3694       twice_nat = 1;
3695       addresses = sm->twice_nat_addresses;
3696       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3697         goto match;
3698     }
3699
3700   return;
3701
3702 match:
3703   if (!is_delete)
3704     {
3705       /* Don't trip over lease renewal, static config */
3706       for (j = 0; j < vec_len (addresses); j++)
3707         if (addresses[j].addr.as_u32 == address->as_u32)
3708           return;
3709
3710       (void) snat_add_address (sm, address, ~0, twice_nat);
3711       /* Scan static map resolution vector */
3712       for (j = 0; j < vec_len (sm->to_resolve); j++)
3713         {
3714           rp = sm->to_resolve + j;
3715           if (rp->addr_only)
3716             continue;
3717           /* On this interface? */
3718           if (rp->sw_if_index == sw_if_index)
3719             {
3720               /* Indetity mapping? */
3721               if (rp->l_addr.as_u32 == 0)
3722                 l_addr.as_u32 = address[0].as_u32;
3723               else
3724                 l_addr.as_u32 = rp->l_addr.as_u32;
3725               /* Add the static mapping */
3726               rv = snat_add_static_mapping (
3727                 l_addr, address[0], rp->l_port, rp->e_port, rp->vrf_id,
3728                 rp->addr_only, ~0 /* sw_if_index */, rp->proto, 1,
3729                 rp->twice_nat, rp->out2in_only, rp->tag, rp->identity_nat,
3730                 rp->pool_addr, rp->exact);
3731               if (rv)
3732                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
3733                                     "i4", rv);
3734             }
3735         }
3736       return;
3737     }
3738   else
3739     {
3740       (void) snat_del_address (sm, address[0], 1, twice_nat);
3741       return;
3742     }
3743 }
3744
3745 int
3746 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3747                             u8 twice_nat)
3748 {
3749   ip4_main_t *ip4_main = sm->ip4_main;
3750   ip4_address_t *first_int_addr;
3751   snat_static_map_resolve_t *rp;
3752   u32 *indices_to_delete = 0;
3753   int i, j;
3754   u32 *auto_add_sw_if_indices =
3755     twice_nat ? sm->
3756     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3757
3758   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3759     );
3760
3761   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3762     {
3763       if (auto_add_sw_if_indices[i] == sw_if_index)
3764         {
3765           if (is_del)
3766             {
3767               /* if have address remove it */
3768               if (first_int_addr)
3769                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3770               else
3771                 {
3772                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3773                     {
3774                       rp = sm->to_resolve + j;
3775                       if (rp->sw_if_index == sw_if_index)
3776                         vec_add1 (indices_to_delete, j);
3777                     }
3778                   if (vec_len (indices_to_delete))
3779                     {
3780                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3781                         vec_del1 (sm->to_resolve, j);
3782                       vec_free (indices_to_delete);
3783                     }
3784                 }
3785               if (twice_nat)
3786                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3787               else
3788                 vec_del1 (sm->auto_add_sw_if_indices, i);
3789             }
3790           else
3791             return VNET_API_ERROR_VALUE_EXIST;
3792
3793           return 0;
3794         }
3795     }
3796
3797   if (is_del)
3798     return VNET_API_ERROR_NO_SUCH_ENTRY;
3799
3800   /* add to the auto-address list */
3801   if (twice_nat)
3802     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3803   else
3804     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3805
3806   /* If the address is already bound - or static - add it now */
3807   if (first_int_addr)
3808     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3809
3810   return 0;
3811 }
3812
3813 int
3814 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3815                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3816                       u32 vrf_id, int is_in)
3817 {
3818   ip4_header_t ip;
3819   clib_bihash_kv_16_8_t kv, value;
3820   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3821   snat_session_t *s;
3822   snat_main_per_thread_data_t *tsm;
3823
3824   if (!sm->endpoint_dependent)
3825     return VNET_API_ERROR_FEATURE_DISABLED;
3826
3827   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3828   if (sm->num_workers > 1)
3829     tsm =
3830       vec_elt_at_index (sm->per_thread_data,
3831                         sm->worker_in2out_cb (&ip, fib_index, 0));
3832   else
3833     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3834
3835   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
3836   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3837     {
3838       return VNET_API_ERROR_NO_SUCH_ENTRY;
3839     }
3840
3841   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3842     return VNET_API_ERROR_UNSPECIFIED;
3843   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3844   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3845   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3846   return 0;
3847 }
3848
3849 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3850                                  vlib_node_runtime_t * node,
3851                                  vlib_frame_t * frame)
3852 {
3853   return 0;
3854 }
3855
3856 /* *INDENT-OFF* */
3857 VLIB_REGISTER_NODE (nat_default_node) = {
3858   .name = "nat-default",
3859   .vector_size = sizeof (u32),
3860   .format_trace = 0,
3861   .type = VLIB_NODE_TYPE_INTERNAL,
3862   .n_errors = 0,
3863   .n_next_nodes = NAT_N_NEXT,
3864   .next_nodes = {
3865     [NAT_NEXT_DROP] = "error-drop",
3866     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3867     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3868     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3869     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3870     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3871     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3872     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3873     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3874     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3875   },
3876 };
3877 /* *INDENT-ON* */
3878
3879 void
3880 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3881 {
3882   f->l3_csum_delta = 0;
3883   f->l4_csum_delta = 0;
3884   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3885       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3886     {
3887       f->l3_csum_delta =
3888         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3889       f->l3_csum_delta =
3890         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3891     }
3892   else
3893     {
3894       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3895     }
3896   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3897       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3898     {
3899       f->l3_csum_delta =
3900         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3901       f->l3_csum_delta =
3902         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3903     }
3904   else
3905     {
3906       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3907     }
3908   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3909     {
3910       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3911       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3912     }
3913   else
3914     {
3915       f->rewrite.sport = f->match.sport;
3916     }
3917   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3918     {
3919       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3920       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3921     }
3922   else
3923     {
3924       f->rewrite.dport = f->match.dport;
3925     }
3926   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3927       f->rewrite.icmp_id != f->match.icmp_id)
3928     {
3929       f->l4_csum_delta =
3930         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3931       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.icmp_id);
3932     }
3933   else
3934     {
3935       f->rewrite.icmp_id = f->match.icmp_id;
3936     }
3937   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3938     {
3939     }
3940   else
3941     {
3942       f->rewrite.fib_index = f->match.fib_index;
3943     }
3944 }
3945
3946 static_always_inline int nat_6t_flow_icmp_translate (snat_main_t *sm,
3947                                                      vlib_buffer_t *b,
3948                                                      ip4_header_t *ip,
3949                                                      nat_6t_flow_t *f);
3950
3951 static_always_inline void
3952 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3953                            nat_6t_flow_t *f, nat_protocol_t proto,
3954                            int is_icmp_inner_ip4)
3955 {
3956   udp_header_t *udp = ip4_next_header (ip);
3957   tcp_header_t *tcp = (tcp_header_t *) udp;
3958
3959   if ((NAT_PROTOCOL_TCP == proto || NAT_PROTOCOL_UDP == proto) &&
3960       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3961     {
3962       if (!is_icmp_inner_ip4)
3963         { // regular case
3964           ip->src_address = f->rewrite.saddr;
3965           ip->dst_address = f->rewrite.daddr;
3966           udp->src_port = f->rewrite.sport;
3967           udp->dst_port = f->rewrite.dport;
3968         }
3969       else
3970         { // icmp inner ip4 - reversed saddr/daddr
3971           ip->src_address = f->rewrite.daddr;
3972           ip->dst_address = f->rewrite.saddr;
3973           udp->src_port = f->rewrite.dport;
3974           udp->dst_port = f->rewrite.sport;
3975         }
3976
3977       if (NAT_PROTOCOL_TCP == proto)
3978         {
3979           ip_csum_t tcp_sum = tcp->checksum;
3980           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3981           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3982           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3983           tcp->checksum = ip_csum_fold (tcp_sum);
3984         }
3985       else if (proto == NAT_PROTOCOL_UDP && udp->checksum)
3986         {
3987           ip_csum_t udp_sum = udp->checksum;
3988           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3989           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3990           udp->checksum = ip_csum_fold (udp_sum);
3991         }
3992     }
3993   else
3994     {
3995       if (!is_icmp_inner_ip4)
3996         { // regular case
3997           ip->src_address = f->rewrite.saddr;
3998           ip->dst_address = f->rewrite.daddr;
3999         }
4000       else
4001         { // icmp inner ip4 - reversed saddr/daddr
4002           ip->src_address = f->rewrite.daddr;
4003           ip->dst_address = f->rewrite.saddr;
4004         }
4005     }
4006
4007   ip_csum_t ip_sum = ip->checksum;
4008   ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
4009   ip->checksum = ip_csum_fold (ip_sum);
4010   ASSERT (ip->checksum == ip4_header_checksum (ip));
4011 }
4012
4013 static_always_inline int
4014 nat_6t_flow_icmp_translate (snat_main_t *sm, vlib_buffer_t *b,
4015                             ip4_header_t *ip, nat_6t_flow_t *f)
4016 {
4017   if (IP_PROTOCOL_ICMP != ip->protocol)
4018     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4019
4020   icmp46_header_t *icmp = ip4_next_header (ip);
4021   icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
4022
4023   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
4024     {
4025       if (icmp->checksum == 0)
4026         icmp->checksum = 0xffff;
4027
4028       if (!icmp_type_is_error_message (icmp->type))
4029         {
4030           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
4031               (f->rewrite.icmp_id != echo->identifier))
4032             {
4033               ip_csum_t sum = icmp->checksum;
4034               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
4035                                     icmp_echo_header_t,
4036                                     identifier /* changed member */);
4037               echo->identifier = f->rewrite.icmp_id;
4038               icmp->checksum = ip_csum_fold (sum);
4039             }
4040         }
4041       else
4042         {
4043           // errors are not fragmented
4044           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
4045
4046           if (!ip4_header_checksum_is_valid (inner_ip))
4047             {
4048               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4049             }
4050
4051           nat_protocol_t inner_proto =
4052             ip_proto_to_nat_proto (inner_ip->protocol);
4053
4054           ip_csum_t icmp_sum = icmp->checksum;
4055
4056           switch (inner_proto)
4057             {
4058             case NAT_PROTOCOL_UDP:
4059             case NAT_PROTOCOL_TCP:
4060               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
4061                                          1 /* is_icmp_inner_ip4 */);
4062               icmp_sum = ip_csum_sub_even (icmp_sum, f->l3_csum_delta);
4063               icmp->checksum = ip_csum_fold (icmp_sum);
4064               break;
4065             case NAT_PROTOCOL_ICMP:
4066               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
4067                 {
4068                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
4069                   icmp_echo_header_t *inner_echo =
4070                     (icmp_echo_header_t *) (inner_icmp + 1);
4071                   if (f->rewrite.icmp_id != inner_echo->identifier)
4072                     {
4073                       ip_csum_t sum = icmp->checksum;
4074                       sum = ip_csum_update (
4075                         sum, inner_echo->identifier, f->rewrite.icmp_id,
4076                         icmp_echo_header_t, identifier /* changed member */);
4077                       icmp->checksum = ip_csum_fold (sum);
4078                       ip_csum_t inner_sum = inner_icmp->checksum;
4079                       inner_sum = ip_csum_update (
4080                         sum, inner_echo->identifier, f->rewrite.icmp_id,
4081                         icmp_echo_header_t, identifier /* changed member */);
4082                       inner_icmp->checksum = ip_csum_fold (inner_sum);
4083                       inner_echo->identifier = f->rewrite.icmp_id;
4084                     }
4085                 }
4086               break;
4087             default:
4088               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
4089               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4090             }
4091         }
4092     }
4093   return NAT_ED_TRNSL_ERR_SUCCESS;
4094 }
4095
4096 nat_translation_error_e
4097 nat_6t_flow_buf_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
4098                            nat_6t_flow_t *f, nat_protocol_t proto,
4099                            int is_output_feature)
4100 {
4101   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
4102     {
4103       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
4104     }
4105
4106   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */);
4107
4108   if (NAT_PROTOCOL_ICMP == proto)
4109     {
4110       return nat_6t_flow_icmp_translate (sm, b, ip, f);
4111     }
4112
4113   return NAT_ED_TRNSL_ERR_SUCCESS;
4114 }
4115
4116 u8 *
4117 format_nat_6t (u8 *s, va_list *args)
4118 {
4119   nat_6t_t *t = va_arg (*args, nat_6t_t *);
4120
4121   s = format (s, "saddr %U sport %u daddr %U dport %u proto %U fib_idx %u",
4122               format_ip4_address, t->saddr.as_u8,
4123               clib_net_to_host_u16 (t->sport), format_ip4_address,
4124               t->daddr.as_u8, clib_net_to_host_u16 (t->dport),
4125               format_ip_protocol, t->proto, t->fib_index);
4126   return s;
4127 }
4128
4129 u8 *
4130 format_nat_ed_translation_error (u8 *s, va_list *args)
4131 {
4132   nat_translation_error_e e = va_arg (*args, nat_translation_error_e);
4133
4134   switch (e)
4135     {
4136     case NAT_ED_TRNSL_ERR_SUCCESS:
4137       s = format (s, "success");
4138       break;
4139     case NAT_ED_TRNSL_ERR_TRANSLATION_FAILED:
4140       s = format (s, "translation-failed");
4141       break;
4142     case NAT_ED_TRNSL_ERR_FLOW_MISMATCH:
4143       s = format (s, "flow-mismatch");
4144       break;
4145     }
4146   return s;
4147 }
4148
4149 u8 *
4150 format_nat_6t_flow (u8 *s, va_list *args)
4151 {
4152   nat_6t_flow_t *f = va_arg (*args, nat_6t_flow_t *);
4153
4154   s = format (s, "match: %U ", format_nat_6t, &f->match);
4155   int r = 0;
4156   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE)
4157     {
4158       s = format (s, "rewrite: saddr %U ", format_ip4_address,
4159                   f->rewrite.saddr.as_u8);
4160       r = 1;
4161     }
4162   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE)
4163     {
4164       if (!r)
4165         {
4166           s = format (s, "rewrite: ");
4167           r = 1;
4168         }
4169       s = format (s, "sport %u ", clib_net_to_host_u16 (f->rewrite.sport));
4170     }
4171   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE)
4172     {
4173       if (!r)
4174         {
4175           s = format (s, "rewrite: ");
4176           r = 1;
4177         }
4178       s = format (s, "daddr %U ", format_ip4_address, f->rewrite.daddr.as_u8);
4179     }
4180   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE)
4181     {
4182       if (!r)
4183         {
4184           s = format (s, "rewrite: ");
4185           r = 1;
4186         }
4187       s = format (s, "dport %u ", clib_net_to_host_u16 (f->rewrite.dport));
4188     }
4189   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
4190     {
4191       if (!r)
4192         {
4193           s = format (s, "rewrite: ");
4194           r = 1;
4195         }
4196       s = format (s, "icmp-id %u ", clib_net_to_host_u16 (f->rewrite.icmp_id));
4197     }
4198   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
4199     {
4200       if (!r)
4201         {
4202           s = format (s, "rewrite: ");
4203           r = 1;
4204         }
4205       s = format (s, "txfib %u ", f->rewrite.fib_index);
4206     }
4207   return s;
4208 }
4209
4210 /*
4211  * fd.io coding-style-patch-verification: ON
4212  *
4213  * Local Variables:
4214  * eval: (c-set-style "gnu")
4215  * End:
4216  */