src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/lib/ipfix_logging.h>
  25 #include <nat/lib/nat_syslog.h>
  26 #include <nat/nat_inlines.h>
  27 #include <nat/nat44/inlines.h>
  28 #include <nat/nat_affinity.h>
  29 #include <vnet/fib/fib_table.h>
  30 #include <vnet/fib/ip4_fib.h>
  31 #include <vnet/ip/reass/ip4_sv_reass.h>
  32 #include <vppinfra/bihash_16_8.h>
  33 #include <nat/nat44/ed_inlines.h>
  34 #include <vnet/ip/ip_table.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_ha.h>
  37 #include <nat/nat44-ei/nat44_ei.h>
  38
  39 #include <vpp/app/version.h>
  40 #include <nat/lib/nat_inlines.h>
  41
  42 snat_main_t snat_main;
  43
  44 #define skip_if_disabled()                                                    \
  45   do                                                                          \
  46     {                                                                         \
  47       snat_main_t *sm = &snat_main;                                           \
  48       if (PREDICT_FALSE (!sm->enabled))                                       \
  49         return;                                                               \
  50     }                                                                         \
  51   while (0)
  52
  53 #define fail_if_enabled()                                                     \
  54   do                                                                          \
  55     {                                                                         \
  56       snat_main_t *sm = &snat_main;                                           \
  57       if (PREDICT_FALSE (sm->enabled))                                        \
  58         {                                                                     \
  59           nat_log_err ("plugin enabled");                                     \
  60           return 1;                                                           \
  61         }                                                                     \
  62     }                                                                         \
  63   while (0)
  64
  65 #define fail_if_disabled()                                                    \
  66   do                                                                          \
  67     {                                                                         \
  68       snat_main_t *sm = &snat_main;                                           \
  69       if (PREDICT_FALSE (!sm->enabled))                                       \
  70         {                                                                     \
  71           nat_log_err ("plugin disabled");                                    \
  72           return 1;                                                           \
  73         }                                                                     \
  74     }                                                                         \
  75   while (0)
  76
  77 /* *INDENT-OFF* */
  78 /* Hook up input features */
  79 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat-pre-in2out",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  83                                "ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat-pre-out2in",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  89                                "ip4-dhcp-client-detect",
  90                                "ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-in2out-worker-handoff",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  96 };
  97 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-out2in-worker-handoff",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 101                                "ip4-dhcp-client-detect"),
 102 };
 103 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-in2out",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-out2in",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 112                                "ip4-dhcp-client-detect"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-classify",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 118 };
 119 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-ed-in2out",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 123 };
 124 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-ed-out2in",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 128                                "ip4-dhcp-client-detect"),
 129 };
 130 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 131   .arc_name = "ip4-unicast",
 132   .node_name = "nat44-ed-classify",
 133   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 134 };
 135 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 136   .arc_name = "ip4-unicast",
 137   .node_name = "nat44-handoff-classify",
 138   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 141   .arc_name = "ip4-unicast",
 142   .node_name = "nat44-in2out-fast",
 143   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 144 };
 145 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 146   .arc_name = "ip4-unicast",
 147   .node_name = "nat44-out2in-fast",
 148   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 149                                "ip4-dhcp-client-detect"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 152   .arc_name = "ip4-unicast",
 153   .node_name = "nat44-hairpin-dst",
 154   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 155 };
 156
 157 /* Hook up output features */
 158 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 159   .arc_name = "ip4-output",
 160   .node_name = "nat44-in2out-output",
 161   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 162 };
 163 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 164   .arc_name = "ip4-output",
 165   .node_name = "nat44-in2out-output-worker-handoff",
 166   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 167 };
 168 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 169   .arc_name = "ip4-output",
 170   .node_name = "nat44-hairpin-src",
 171   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 172 };
 173 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 174   .arc_name = "ip4-output",
 175   .node_name = "nat-pre-in2out-output",
 176   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 177   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 178 };
 179 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 180   .arc_name = "ip4-output",
 181   .node_name = "nat44-ed-in2out-output",
 182   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 183   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 184 };
 185
 186 /* Hook up ip4-local features */
 187 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 188 {
 189   .arc_name = "ip4-local",
 190   .node_name = "nat44-hairpinning",
 191   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 192 };
 193
 194 VLIB_PLUGIN_REGISTER () = {
 195     .version = VPP_BUILD_VER,
 196     .description = "Network Address Translation (NAT)",
 197 };
 198 /* *INDENT-ON* */
 199
 200 static void nat44_ed_db_init (u32 translations, u32 translation_buckets,
 201                               u32 user_buckets);
 202
 203 static void nat44_ed_db_free ();
 204
 205 static u32
 206 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
 207                                u32 rx_fib_index, u8 is_output);
 208
 209 static u32
 210 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
 211                                u8 is_output);
 212
 213 u32 nat_calc_bihash_buckets (u32 n_elts);
 214
 215 u8 *
 216 format_session_kvp (u8 * s, va_list * args)
 217 {
 218   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 219
 220   s = format (s, "%U thread-index %llu session-index %llu", format_snat_key,
 221               v->key, nat_value_get_thread_index (v),
 222               nat_value_get_session_index (v));
 223
 224   return s;
 225 }
 226
 227 u8 *
 228 format_static_mapping_kvp (u8 * s, va_list * args)
 229 {
 230   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 231
 232   s = format (s, "%U static-mapping-index %llu",
 233               format_snat_key, v->key, v->value);
 234
 235   return s;
 236 }
 237
 238 u8 *
 239 format_user_kvp (u8 * s, va_list * args)
 240 {
 241   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 242   snat_user_key_t k;
 243
 244   k.as_u64 = v->key;
 245
 246   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
 247               k.fib_index, v->value);
 248
 249   return s;
 250 }
 251
 252 u8 *
 253 format_ed_session_kvp (u8 * s, va_list * args)
 254 {
 255   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 256
 257   u8 proto;
 258   u16 r_port, l_port;
 259   ip4_address_t l_addr, r_addr;
 260   u32 fib_index;
 261
 262   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 263   s = format (s,
 264               "local %U:%d remote %U:%d proto %U fib %d thread-index %u "
 265               "session-index %u",
 266               format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 267               format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 268               format_ip_protocol, proto, fib_index,
 269               ed_value_get_thread_index (v), ed_value_get_session_index (v));
 270
 271   return s;
 272 }
 273
 274 void
 275 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 276                        u8 is_ha)
 277 {
 278   clib_bihash_kv_8_8_t kv;
 279
 280   if (is_ed_session (s))
 281     {
 282       per_vrf_sessions_unregister_session (s, thread_index);
 283
 284       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 285         nat_elog_warn ("flow hash del failed");
 286
 287       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 288         nat_elog_warn ("flow hash del failed");
 289     }
 290
 291   if (is_fwd_bypass_session (s))
 292     {
 293       return;
 294     }
 295
 296   /* session lookup tables */
 297   if (is_ed_session (s))
 298     {
 299       if (is_affinity_sessions (s))
 300         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 301                              s->nat_proto, s->out2in.port);
 302
 303       if (!is_ha)
 304         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 305                                &s->in2out.addr, s->in2out.port,
 306                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 307                                &s->out2in.addr, s->out2in.port,
 308                                &s->ext_host_addr, s->ext_host_port,
 309                                s->nat_proto, is_twice_nat_session (s));
 310     }
 311   else
 312     {
 313       init_nat_i2o_k (&kv, s);
 314       if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
 315         nat_elog_warn ("in2out key del failed");
 316       init_nat_o2i_k (&kv, s);
 317       if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
 318         nat_elog_warn ("out2in key del failed");
 319
 320       if (!is_ha)
 321         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 322                                  &s->in2out.addr, s->in2out.port,
 323                                  &s->out2in.addr, s->out2in.port,
 324                                  s->nat_proto);
 325     }
 326
 327   if (snat_is_unk_proto_session (s))
 328     return;
 329
 330   if (!is_ha)
 331     {
 332       /* log NAT event */
 333       nat_ipfix_logging_nat44_ses_delete (thread_index,
 334                                           s->in2out.addr.as_u32,
 335                                           s->out2in.addr.as_u32,
 336                                           s->nat_proto,
 337                                           s->in2out.port,
 338                                           s->out2in.port,
 339                                           s->in2out.fib_index);
 340
 341       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 342                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 343                    thread_index);
 344     }
 345
 346   /* Twice NAT address and port for external host */
 347   if (is_twice_nat_session (s))
 348     {
 349       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 350                                           thread_index,
 351                                           &s->ext_host_nat_addr,
 352                                           s->ext_host_nat_port, s->nat_proto);
 353     }
 354
 355   if (snat_is_session_static (s))
 356     return;
 357
 358   snat_free_outside_address_and_port (sm->addresses, thread_index,
 359                                       &s->out2in.addr, s->out2in.port,
 360                                       s->nat_proto);
 361 }
 362
 363 snat_user_t *
 364 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 365                         u32 thread_index)
 366 {
 367   snat_user_t *u = 0;
 368   snat_user_key_t user_key;
 369   clib_bihash_kv_8_8_t kv, value;
 370   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 371   dlist_elt_t *per_user_list_head_elt;
 372
 373   user_key.addr.as_u32 = addr->as_u32;
 374   user_key.fib_index = fib_index;
 375   kv.key = user_key.as_u64;
 376
 377   /* Ever heard of the "user" = src ip4 address before? */
 378   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 379     {
 380       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 381         {
 382           vlib_increment_simple_counter (&sm->user_limit_reached,
 383                                          thread_index, 0, 1);
 384           nat_elog_warn ("maximum user limit reached");
 385           return NULL;
 386         }
 387       /* no, make a new one */
 388       pool_get (tsm->users, u);
 389       clib_memset (u, 0, sizeof (*u));
 390
 391       u->addr.as_u32 = addr->as_u32;
 392       u->fib_index = fib_index;
 393
 394       pool_get (tsm->list_pool, per_user_list_head_elt);
 395
 396       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 397         tsm->list_pool;
 398
 399       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 400
 401       kv.value = u - tsm->users;
 402
 403       /* add user */
 404       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 405         {
 406           nat_elog_warn ("user_hash key add failed");
 407           nat44_delete_user_with_no_session (sm, u, thread_index);
 408           return NULL;
 409         }
 410
 411       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 412                                pool_elts (tsm->users));
 413     }
 414   else
 415     {
 416       u = pool_elt_at_index (tsm->users, value.value);
 417     }
 418
 419   return u;
 420 }
 421
 422 // only NAT EI
 423 snat_session_t *
 424 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 425                               u32 thread_index, f64 now)
 426 {
 427   snat_session_t *s;
 428   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 429   u32 oldest_per_user_translation_list_index, session_index;
 430   dlist_elt_t *oldest_per_user_translation_list_elt;
 431   dlist_elt_t *per_user_translation_list_elt;
 432
 433   /* Over quota? Recycle the least recently used translation */
 434   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 435     {
 436       oldest_per_user_translation_list_index =
 437         clib_dlist_remove_head (tsm->list_pool,
 438                                 u->sessions_per_user_list_head_index);
 439
 440       ASSERT (oldest_per_user_translation_list_index != ~0);
 441
 442       /* Add it back to the end of the LRU list */
 443       clib_dlist_addtail (tsm->list_pool,
 444                           u->sessions_per_user_list_head_index,
 445                           oldest_per_user_translation_list_index);
 446       /* Get the list element */
 447       oldest_per_user_translation_list_elt =
 448         pool_elt_at_index (tsm->list_pool,
 449                            oldest_per_user_translation_list_index);
 450
 451       /* Get the session index from the list element */
 452       session_index = oldest_per_user_translation_list_elt->value;
 453
 454       /* Get the session */
 455       s = pool_elt_at_index (tsm->sessions, session_index);
 456
 457       // TODO: ONLY EI version should be called
 458       nat_free_session_data (sm, s, thread_index, 0);
 459       if (snat_is_session_static (s))
 460         u->nstaticsessions--;
 461       else
 462         u->nsessions--;
 463       s->flags = 0;
 464       s->total_bytes = 0;
 465       s->total_pkts = 0;
 466       s->state = 0;
 467       s->ext_host_addr.as_u32 = 0;
 468       s->ext_host_port = 0;
 469       s->ext_host_nat_addr.as_u32 = 0;
 470       s->ext_host_nat_port = 0;
 471     }
 472   else
 473     {
 474       pool_get (tsm->sessions, s);
 475       clib_memset (s, 0, sizeof (*s));
 476
 477       /* Create list elts */
 478       pool_get (tsm->list_pool, per_user_translation_list_elt);
 479       clib_dlist_init (tsm->list_pool,
 480                        per_user_translation_list_elt - tsm->list_pool);
 481
 482       per_user_translation_list_elt->value = s - tsm->sessions;
 483       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 484       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 485
 486       clib_dlist_addtail (tsm->list_pool,
 487                           s->per_user_list_head_index,
 488                           per_user_translation_list_elt - tsm->list_pool);
 489
 490       s->user_index = u - tsm->users;
 491       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 492                                pool_elts (tsm->sessions));
 493     }
 494
 495   s->ha_last_refreshed = now;
 496
 497   return s;
 498 }
 499
 500 void
 501 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 502                           int is_add)
 503 {
 504   snat_main_t *sm = &snat_main;
 505   fib_prefix_t prefix = {
 506     .fp_len = p_len,
 507     .fp_proto = FIB_PROTOCOL_IP4,
 508     .fp_addr = {
 509                 .ip4.as_u32 = addr->as_u32,
 510                 },
 511   };
 512   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 513
 514   if (is_add)
 515     fib_table_entry_update_one_path (fib_index,
 516                                      &prefix,
 517                                      sm->fib_src_low,
 518                                      (FIB_ENTRY_FLAG_CONNECTED |
 519                                       FIB_ENTRY_FLAG_LOCAL |
 520                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 521                                      DPO_PROTO_IP4,
 522                                      NULL,
 523                                      sw_if_index,
 524                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 525   else
 526     fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 527 }
 528
 529 int
 530 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 531                   u8 twice_nat)
 532 {
 533   snat_address_t *ap;
 534   snat_interface_t *i;
 535   vlib_thread_main_t *tm = vlib_get_thread_main ();
 536
 537   if (twice_nat && !sm->endpoint_dependent)
 538     {
 539       nat_log_err ("unsupported");
 540       return VNET_API_ERROR_UNSUPPORTED;
 541     }
 542
 543   /* Check if address already exists */
 544   /* *INDENT-OFF* */
 545   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 546     {
 547       if (ap->addr.as_u32 == addr->as_u32)
 548         {
 549           nat_log_err ("address exist");
 550           return VNET_API_ERROR_VALUE_EXIST;
 551         }
 552     }
 553   /* *INDENT-ON* */
 554
 555   if (twice_nat)
 556     vec_add2 (sm->twice_nat_addresses, ap, 1);
 557   else
 558     vec_add2 (sm->addresses, ap, 1);
 559
 560   ap->addr = *addr;
 561   if (vrf_id != ~0)
 562     ap->fib_index =
 563       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 564                                          sm->fib_src_low);
 565   else
 566     ap->fib_index = ~0;
 567
 568   /* *INDENT-OFF* */
 569   #define _(N, i, n, s) \
 570     clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 571     ap->busy_##n##_ports = 0; \
 572     ap->busy_##n##_ports_per_thread = 0;\
 573     vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 574     foreach_nat_protocol
 575   #undef _
 576   /* *INDENT-ON* */
 577
 578   if (twice_nat)
 579     return 0;
 580
 581   /* Add external address to FIB */
 582   /* *INDENT-OFF* */
 583   pool_foreach (i, sm->interfaces)
 584    {
 585     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 586       continue;
 587
 588     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 589     break;
 590   }
 591   pool_foreach (i, sm->output_feature_interfaces)
 592    {
 593     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 594       continue;
 595
 596     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 597     break;
 598   }
 599   /* *INDENT-ON* */
 600
 601   return 0;
 602 }
 603
 604 static int
 605 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 606 {
 607   snat_static_mapping_t *m;
 608   /* *INDENT-OFF* */
 609   pool_foreach (m, sm->static_mappings)
 610    {
 611       if (is_addr_only_static_mapping (m) ||
 612           is_out2in_only_static_mapping (m) ||
 613           is_identity_static_mapping (m))
 614         continue;
 615       if (m->external_addr.as_u32 == addr.as_u32)
 616         return 1;
 617   }
 618   /* *INDENT-ON* */
 619
 620   return 0;
 621 }
 622
 623 static void
 624 snat_add_static_mapping_when_resolved (snat_main_t *sm, ip4_address_t l_addr,
 625                                        u16 l_port, u32 sw_if_index, u16 e_port,
 626                                        u32 vrf_id, nat_protocol_t proto,
 627                                        int addr_only, u8 *tag, int twice_nat,
 628                                        int out2in_only, int identity_nat,
 629                                        ip4_address_t pool_addr, int exact)
 630 {
 631   snat_static_map_resolve_t *rp;
 632
 633   vec_add2 (sm->to_resolve, rp, 1);
 634   rp->l_addr.as_u32 = l_addr.as_u32;
 635   rp->l_port = l_port;
 636   rp->sw_if_index = sw_if_index;
 637   rp->e_port = e_port;
 638   rp->vrf_id = vrf_id;
 639   rp->proto = proto;
 640   rp->addr_only = addr_only;
 641   rp->twice_nat = twice_nat;
 642   rp->out2in_only = out2in_only;
 643   rp->identity_nat = identity_nat;
 644   rp->tag = vec_dup (tag);
 645   rp->pool_addr = pool_addr;
 646   rp->exact = exact;
 647 }
 648
 649 u32
 650 get_thread_idx_by_port (u16 e_port)
 651 {
 652   snat_main_t *sm = &snat_main;
 653   u32 thread_idx = sm->num_workers;
 654   if (sm->num_workers > 1)
 655     {
 656       thread_idx =
 657         sm->first_worker_index +
 658         sm->workers[(e_port - 1024) / sm->port_per_thread];
 659     }
 660   return thread_idx;
 661 }
 662
 663 void
 664 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 665                                     snat_main_per_thread_data_t * tsm,
 666                                     ip4_address_t l_addr,
 667                                     u16 l_port,
 668                                     u8 protocol,
 669                                     u32 fib_index, int addr_only,
 670                                     ip4_address_t e_addr, u16 e_port)
 671 {
 672   snat_session_t *s;
 673   u32 *indexes_to_free = NULL;
 674   /* *INDENT-OFF* */
 675   pool_foreach (s, tsm->sessions) {
 676     if (s->in2out.fib_index != fib_index ||
 677         s->in2out.addr.as_u32 != l_addr.as_u32)
 678       {
 679         continue;
 680       }
 681     if (!addr_only)
 682       {
 683         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 684             s->out2in.port != e_port ||
 685             s->in2out.port != l_port ||
 686             s->nat_proto != protocol)
 687           continue;
 688       }
 689
 690     if (is_lb_session (s))
 691       continue;
 692     if (!snat_is_session_static (s))
 693       continue;
 694     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 695     vec_add1 (indexes_to_free, s - tsm->sessions);
 696     if (!addr_only)
 697       break;
 698   }
 699   /* *INDENT-ON* */
 700   u32 *ses_index;
 701   vec_foreach (ses_index, indexes_to_free)
 702   {
 703     s = pool_elt_at_index (tsm->sessions, *ses_index);
 704     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 705   }
 706   vec_free (indexes_to_free);
 707 }
 708
 709 static_always_inline int
 710 nat44_ed_add_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 711                                  u16 l_port, u16 e_port, u32 vrf_id,
 712                                  int addr_only, u32 sw_if_index,
 713                                  nat_protocol_t proto, int is_add,
 714                                  twice_nat_type_t twice_nat, u8 out2in_only,
 715                                  u8 *tag, u8 identity_nat,
 716                                  ip4_address_t pool_addr, int exact)
 717 {
 718   snat_main_t *sm = &snat_main;
 719   snat_static_mapping_t *m;
 720   clib_bihash_kv_8_8_t kv, value;
 721   snat_address_t *a = 0;
 722   u32 fib_index = ~0;
 723   snat_interface_t *interface;
 724   int i;
 725   snat_main_per_thread_data_t *tsm;
 726   snat_user_key_t u_key;
 727   snat_user_t *u;
 728   dlist_elt_t *head, *elt;
 729   u32 elt_index, head_index;
 730   u32 ses_index;
 731   u64 user_index;
 732   snat_session_t *s;
 733   snat_static_map_resolve_t *rp, *rp_match = 0;
 734   nat44_lb_addr_port_t *local;
 735   u32 find = ~0;
 736
 737   if (!sm->endpoint_dependent)
 738     {
 739       if (twice_nat || out2in_only)
 740         return VNET_API_ERROR_UNSUPPORTED;
 741     }
 742
 743   /* If the external address is a specific interface address */
 744   if (sw_if_index != ~0)
 745     {
 746       ip4_address_t *first_int_addr;
 747
 748       for (i = 0; i < vec_len (sm->to_resolve); i++)
 749         {
 750           rp = sm->to_resolve + i;
 751           if (rp->sw_if_index != sw_if_index ||
 752               rp->l_addr.as_u32 != l_addr.as_u32 ||
 753               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 754             continue;
 755
 756           if (!addr_only)
 757             {
 758               if ((rp->l_port != l_port && rp->e_port != e_port)
 759                   || rp->proto != proto)
 760                 continue;
 761             }
 762
 763           rp_match = rp;
 764           break;
 765         }
 766
 767       /* Might be already set... */
 768       first_int_addr = ip4_interface_first_address
 769         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 770
 771       if (is_add)
 772         {
 773           if (rp_match)
 774             return VNET_API_ERROR_VALUE_EXIST;
 775
 776           snat_add_static_mapping_when_resolved (
 777             sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto, addr_only,
 778             tag, twice_nat, out2in_only, identity_nat, pool_addr, exact);
 779
 780           /* DHCP resolution required? */
 781           if (first_int_addr == 0)
 782             {
 783               return 0;
 784             }
 785           else
 786             {
 787               e_addr.as_u32 = first_int_addr->as_u32;
 788               /* Identity mapping? */
 789               if (l_addr.as_u32 == 0)
 790                 l_addr.as_u32 = e_addr.as_u32;
 791             }
 792         }
 793       else
 794         {
 795           if (!rp_match)
 796             return VNET_API_ERROR_NO_SUCH_ENTRY;
 797
 798           vec_del1 (sm->to_resolve, i);
 799
 800           if (first_int_addr)
 801             {
 802               e_addr.as_u32 = first_int_addr->as_u32;
 803               /* Identity mapping? */
 804               if (l_addr.as_u32 == 0)
 805                 l_addr.as_u32 = e_addr.as_u32;
 806             }
 807           else
 808             return 0;
 809         }
 810     }
 811
 812   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 813   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 814     m = 0;
 815   else
 816     m = pool_elt_at_index (sm->static_mappings, value.value);
 817
 818   if (is_add)
 819     {
 820       if (m)
 821         {
 822           if (is_identity_static_mapping (m))
 823             {
 824               /* *INDENT-OFF* */
 825               pool_foreach (local, m->locals)
 826                {
 827                 if (local->vrf_id == vrf_id)
 828                   return VNET_API_ERROR_VALUE_EXIST;
 829               }
 830               /* *INDENT-ON* */
 831               pool_get (m->locals, local);
 832               local->vrf_id = vrf_id;
 833               local->fib_index =
 834                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 835                                                    sm->fib_src_low);
 836               init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
 837                            m->proto, 0, m - sm->static_mappings);
 838               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 839               return 0;
 840             }
 841           else
 842             return VNET_API_ERROR_VALUE_EXIST;
 843         }
 844
 845       if (twice_nat && addr_only)
 846         return VNET_API_ERROR_UNSUPPORTED;
 847
 848       /* Convert VRF id to FIB index */
 849       if (vrf_id != ~0)
 850         fib_index =
 851           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 852                                              sm->fib_src_low);
 853       /* If not specified use inside VRF id from SNAT plugin startup config */
 854       else
 855         {
 856           fib_index = sm->inside_fib_index;
 857           vrf_id = sm->inside_vrf_id;
 858           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 859         }
 860
 861       if (!(out2in_only || identity_nat))
 862         {
 863           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
 864                       addr_only ? 0 : proto);
 865           if (!clib_bihash_search_8_8
 866               (&sm->static_mapping_by_local, &kv, &value))
 867             return VNET_API_ERROR_VALUE_EXIST;
 868         }
 869
 870       /* Find external address in allocated addresses and reserve port for
 871          address and port pair mapping when dynamic translations enabled */
 872       if (!(addr_only || sm->static_mapping_only || out2in_only))
 873         {
 874           for (i = 0; i < vec_len (sm->addresses); i++)
 875             {
 876               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 877                 {
 878                   a = sm->addresses + i;
 879                   /* External port must be unused */
 880                   switch (proto)
 881                     {
 882 #define _(N, j, n, s) \
 883                     case NAT_PROTOCOL_##N: \
 884                       if (a->busy_##n##_port_refcounts[e_port]) \
 885                         return VNET_API_ERROR_INVALID_VALUE; \
 886                       ++a->busy_##n##_port_refcounts[e_port]; \
 887                       if (e_port > 1024) \
 888                         { \
 889                           a->busy_##n##_ports++; \
 890                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 891                         } \
 892                       break;
 893                       foreach_nat_protocol
 894 #undef _
 895                     default:
 896                       nat_elog_info ("unknown protocol");
 897                       return VNET_API_ERROR_INVALID_VALUE_2;
 898                     }
 899                   break;
 900                 }
 901             }
 902           /* External address must be allocated */
 903           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 904             {
 905               if (sw_if_index != ~0)
 906                 {
 907                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 908                     {
 909                       rp = sm->to_resolve + i;
 910                       if (rp->addr_only)
 911                         continue;
 912                       if (rp->sw_if_index != sw_if_index &&
 913                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 914                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 915                           rp->e_port != e_port && rp->proto != proto)
 916                         continue;
 917
 918                       vec_del1 (sm->to_resolve, i);
 919                       break;
 920                     }
 921                 }
 922               return VNET_API_ERROR_NO_SUCH_ENTRY;
 923             }
 924         }
 925
 926       pool_get (sm->static_mappings, m);
 927       clib_memset (m, 0, sizeof (*m));
 928       m->tag = vec_dup (tag);
 929       m->local_addr = l_addr;
 930       m->external_addr = e_addr;
 931       m->twice_nat = twice_nat;
 932
 933       if (twice_nat == TWICE_NAT && exact)
 934         {
 935           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
 936           m->pool_addr = pool_addr;
 937         }
 938
 939       if (out2in_only)
 940         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 941       if (addr_only)
 942         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 943       if (identity_nat)
 944         {
 945           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 946           pool_get (m->locals, local);
 947           local->vrf_id = vrf_id;
 948           local->fib_index = fib_index;
 949         }
 950       else
 951         {
 952           m->vrf_id = vrf_id;
 953           m->fib_index = fib_index;
 954         }
 955       if (!addr_only)
 956         {
 957           m->local_port = l_port;
 958           m->external_port = e_port;
 959           m->proto = proto;
 960         }
 961
 962       if (sm->num_workers > 1)
 963         {
 964           ip4_header_t ip = {
 965             .src_address = m->local_addr,
 966           };
 967           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 968           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 969         }
 970       else
 971         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 972
 973       if (!out2in_only)
 974         {
 975           init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
 976                        0, m - sm->static_mappings);
 977           clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 978         }
 979
 980       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
 981                    m - sm->static_mappings);
 982       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 983
 984       /* Delete dynamic sessions matching local address (+ local port) */
 985       // TODO: based on type of NAT EI/ED
 986       if (!(sm->static_mapping_only))
 987         {
 988           u_key.addr = m->local_addr;
 989           u_key.fib_index = m->fib_index;
 990           kv.key = u_key.as_u64;
 991           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 992             {
 993               user_index = value.value;
 994               u = pool_elt_at_index (tsm->users, user_index);
 995               if (u->nsessions)
 996                 {
 997                   head_index = u->sessions_per_user_list_head_index;
 998                   head = pool_elt_at_index (tsm->list_pool, head_index);
 999                   elt_index = head->next;
1000                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1001                   ses_index = elt->value;
1002                   while (ses_index != ~0)
1003                     {
1004                       s = pool_elt_at_index (tsm->sessions, ses_index);
1005                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1006                       ses_index = elt->value;
1007
1008                       if (snat_is_session_static (s))
1009                         continue;
1010
1011                       if (!addr_only && s->in2out.port != m->local_port)
1012                         continue;
1013
1014                       nat_free_session_data (sm, s,
1015                                              tsm - sm->per_thread_data, 0);
1016                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1017
1018                       if (!addr_only && !sm->endpoint_dependent)
1019                         break;
1020                     }
1021                 }
1022             }
1023         }
1024     }
1025   else
1026     {
1027       if (!m)
1028         {
1029           if (sw_if_index != ~0)
1030             return 0;
1031           else
1032             return VNET_API_ERROR_NO_SUCH_ENTRY;
1033         }
1034
1035       if (identity_nat)
1036         {
1037           if (vrf_id == ~0)
1038             vrf_id = sm->inside_vrf_id;
1039
1040           /* *INDENT-OFF* */
1041           pool_foreach (local, m->locals)
1042            {
1043             if (local->vrf_id == vrf_id)
1044               find = local - m->locals;
1045           }
1046           /* *INDENT-ON* */
1047           if (find == ~0)
1048             return VNET_API_ERROR_NO_SUCH_ENTRY;
1049
1050           local = pool_elt_at_index (m->locals, find);
1051           fib_index = local->fib_index;
1052           pool_put (m->locals, local);
1053         }
1054       else
1055         fib_index = m->fib_index;
1056
1057       /* Free external address port */
1058       if (!(addr_only || sm->static_mapping_only || out2in_only))
1059         {
1060           for (i = 0; i < vec_len (sm->addresses); i++)
1061             {
1062               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1063                 {
1064                   a = sm->addresses + i;
1065                   switch (proto)
1066                     {
1067 #define _(N, j, n, s) \
1068                     case NAT_PROTOCOL_##N: \
1069                       --a->busy_##n##_port_refcounts[e_port]; \
1070                       if (e_port > 1024) \
1071                         { \
1072                           a->busy_##n##_ports--; \
1073                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1074                         } \
1075                       break;
1076                       foreach_nat_protocol
1077 #undef _
1078                     default:
1079                       nat_elog_info ("unknown protocol");
1080                       return VNET_API_ERROR_INVALID_VALUE_2;
1081                     }
1082                   break;
1083                 }
1084             }
1085         }
1086
1087       if (sm->num_workers > 1)
1088         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1089       else
1090         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1091
1092       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1093       if (!out2in_only)
1094         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1095
1096       /* Delete session(s) for static mapping if exist */
1097       if (!(sm->static_mapping_only) ||
1098           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1099         {
1100           if (sm->endpoint_dependent)
1101             {
1102               nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1103                                                   m->local_port, m->proto,
1104                                                   fib_index, addr_only,
1105                                                   e_addr, e_port);
1106             }
1107           else
1108             {
1109               u_key.addr = m->local_addr;
1110               u_key.fib_index = fib_index;
1111               kv.key = u_key.as_u64;
1112               nat44_ei_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1113                                                     e_addr, e_port);
1114             }
1115         }
1116
1117       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1118       if (pool_elts (m->locals))
1119         return 0;
1120
1121       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1122       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1123
1124       vec_free (m->tag);
1125       vec_free (m->workers);
1126       /* Delete static mapping from pool */
1127       pool_put (sm->static_mappings, m);
1128     }
1129
1130   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1131     return 0;
1132
1133   /* Add/delete external address to FIB */
1134   /* *INDENT-OFF* */
1135   pool_foreach (interface, sm->interfaces)
1136    {
1137     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1138       continue;
1139
1140     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1141     break;
1142   }
1143   pool_foreach (interface, sm->output_feature_interfaces)
1144    {
1145     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1146       continue;
1147
1148     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1149     break;
1150   }
1151   /* *INDENT-ON* */
1152
1153   return 0;
1154 }
1155
1156 int
1157 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
1158                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
1159                          u32 sw_if_index, nat_protocol_t proto, int is_add,
1160                          twice_nat_type_t twice_nat, u8 out2in_only, u8 *tag,
1161                          u8 identity_nat, ip4_address_t pool_addr, int exact)
1162 {
1163   snat_main_t *sm = &snat_main;
1164   int rv;
1165
1166   if (sm->endpoint_dependent)
1167     {
1168       rv = nat44_ed_add_del_static_mapping (
1169         l_addr, e_addr, l_port, e_port, vrf_id, addr_only, sw_if_index, proto,
1170         is_add, twice_nat, out2in_only, tag, identity_nat, pool_addr, exact);
1171     }
1172   else
1173     {
1174       rv = nat44_ei_add_del_static_mapping (
1175         l_addr, e_addr, l_port, e_port, proto, sw_if_index, vrf_id, addr_only,
1176         identity_nat, tag, is_add);
1177     }
1178   return rv;
1179 }
1180
1181 int
1182 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1183                                  nat_protocol_t proto,
1184                                  nat44_lb_addr_port_t * locals, u8 is_add,
1185                                  twice_nat_type_t twice_nat, u8 out2in_only,
1186                                  u8 * tag, u32 affinity)
1187 {
1188   snat_main_t *sm = &snat_main;
1189   snat_static_mapping_t *m;
1190   clib_bihash_kv_8_8_t kv, value;
1191   snat_address_t *a = 0;
1192   int i;
1193   nat44_lb_addr_port_t *local;
1194   snat_main_per_thread_data_t *tsm;
1195   snat_session_t *s;
1196   uword *bitmap = 0;
1197
1198   if (!sm->endpoint_dependent)
1199     return VNET_API_ERROR_UNSUPPORTED;
1200
1201   init_nat_k (&kv, e_addr, e_port, 0, proto);
1202   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1203     m = 0;
1204   else
1205     m = pool_elt_at_index (sm->static_mappings, value.value);
1206
1207   if (is_add)
1208     {
1209       if (m)
1210         return VNET_API_ERROR_VALUE_EXIST;
1211
1212       if (vec_len (locals) < 2)
1213         return VNET_API_ERROR_INVALID_VALUE;
1214
1215       /* Find external address in allocated addresses and reserve port for
1216          address and port pair mapping when dynamic translations enabled */
1217       if (!(sm->static_mapping_only || out2in_only))
1218         {
1219           for (i = 0; i < vec_len (sm->addresses); i++)
1220             {
1221               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1222                 {
1223                   a = sm->addresses + i;
1224                   /* External port must be unused */
1225                   switch (proto)
1226                     {
1227 #define _(N, j, n, s) \
1228                     case NAT_PROTOCOL_##N: \
1229                       if (a->busy_##n##_port_refcounts[e_port]) \
1230                         return VNET_API_ERROR_INVALID_VALUE; \
1231                       ++a->busy_##n##_port_refcounts[e_port]; \
1232                       if (e_port > 1024) \
1233                         { \
1234                           a->busy_##n##_ports++; \
1235                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1236                         } \
1237                       break;
1238                       foreach_nat_protocol
1239 #undef _
1240                     default:
1241                       nat_elog_info ("unknown protocol");
1242                       return VNET_API_ERROR_INVALID_VALUE_2;
1243                     }
1244                   break;
1245                 }
1246             }
1247           /* External address must be allocated */
1248           if (!a)
1249             return VNET_API_ERROR_NO_SUCH_ENTRY;
1250         }
1251
1252       pool_get (sm->static_mappings, m);
1253       clib_memset (m, 0, sizeof (*m));
1254       m->tag = vec_dup (tag);
1255       m->external_addr = e_addr;
1256       m->external_port = e_port;
1257       m->proto = proto;
1258       m->twice_nat = twice_nat;
1259       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1260       if (out2in_only)
1261         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1262       m->affinity = affinity;
1263
1264       if (affinity)
1265         m->affinity_per_service_list_head_index =
1266           nat_affinity_get_per_service_list_head_index ();
1267       else
1268         m->affinity_per_service_list_head_index = ~0;
1269
1270       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
1271                    m - sm->static_mappings);
1272       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1273         {
1274           nat_elog_err ("static_mapping_by_external key add failed");
1275           return VNET_API_ERROR_UNSPECIFIED;
1276         }
1277
1278       for (i = 0; i < vec_len (locals); i++)
1279         {
1280           locals[i].fib_index =
1281             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1282                                                locals[i].vrf_id,
1283                                                sm->fib_src_low);
1284           if (!out2in_only)
1285             {
1286               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1287                            locals[i].fib_index, m->proto, 0,
1288                            m - sm->static_mappings);
1289               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1290             }
1291           locals[i].prefix = (i == 0) ? locals[i].probability :
1292             (locals[i - 1].prefix + locals[i].probability);
1293           pool_get (m->locals, local);
1294           *local = locals[i];
1295           if (sm->num_workers > 1)
1296             {
1297               ip4_header_t ip = {
1298                 .src_address = locals[i].addr,
1299               };
1300               bitmap =
1301                 clib_bitmap_set (bitmap,
1302                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1303                                  1);
1304             }
1305         }
1306
1307       /* Assign workers */
1308       if (sm->num_workers > 1)
1309         {
1310           /* *INDENT-OFF* */
1311           clib_bitmap_foreach (i, bitmap)
1312              {
1313                vec_add1(m->workers, i);
1314             }
1315           /* *INDENT-ON* */
1316         }
1317     }
1318   else
1319     {
1320       if (!m)
1321         return VNET_API_ERROR_NO_SUCH_ENTRY;
1322
1323       if (!is_lb_static_mapping (m))
1324         return VNET_API_ERROR_INVALID_VALUE;
1325
1326       /* Free external address port */
1327       if (!(sm->static_mapping_only || out2in_only))
1328         {
1329           for (i = 0; i < vec_len (sm->addresses); i++)
1330             {
1331               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1332                 {
1333                   a = sm->addresses + i;
1334                   switch (proto)
1335                     {
1336 #define _(N, j, n, s) \
1337                     case NAT_PROTOCOL_##N: \
1338                       --a->busy_##n##_port_refcounts[e_port]; \
1339                       if (e_port > 1024) \
1340                         { \
1341                           a->busy_##n##_ports--; \
1342                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1343                         } \
1344                       break;
1345                       foreach_nat_protocol
1346 #undef _
1347                     default:
1348                       nat_elog_info ("unknown protocol");
1349                       return VNET_API_ERROR_INVALID_VALUE_2;
1350                     }
1351                   break;
1352                 }
1353             }
1354         }
1355
1356       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1357       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1358         {
1359           nat_elog_err ("static_mapping_by_external key del failed");
1360           return VNET_API_ERROR_UNSPECIFIED;
1361         }
1362
1363       /* *INDENT-OFF* */
1364       pool_foreach (local, m->locals)
1365       {
1366           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1367                             sm->fib_src_low);
1368           if (!out2in_only)
1369             {
1370 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1371               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1372                 {
1373                   nat_elog_err ("static_mapping_by_local key del failed");
1374                   return VNET_API_ERROR_UNSPECIFIED;
1375                 }
1376             }
1377
1378           if (sm->num_workers > 1)
1379             {
1380               ip4_header_t ip = {
1381                 .src_address = local->addr,
1382               };
1383               tsm = vec_elt_at_index (sm->per_thread_data,
1384                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1385             }
1386           else
1387             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1388
1389           /* Delete sessions */
1390           pool_foreach (s, tsm->sessions) {
1391             if (!(is_lb_session (s)))
1392               continue;
1393
1394             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1395                 s->in2out.port != local->port)
1396               continue;
1397
1398             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1399             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1400           }
1401       }
1402       /* *INDENT-ON* */
1403       if (m->affinity)
1404         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1405       pool_free (m->locals);
1406       vec_free (m->tag);
1407       vec_free (m->workers);
1408
1409       pool_put (sm->static_mappings, m);
1410     }
1411
1412   return 0;
1413 }
1414
1415 int
1416 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1417                                        ip4_address_t l_addr, u16 l_port,
1418                                        nat_protocol_t proto, u32 vrf_id,
1419                                        u8 probability, u8 is_add)
1420 {
1421   snat_main_t *sm = &snat_main;
1422   snat_static_mapping_t *m = 0;
1423   clib_bihash_kv_8_8_t kv, value;
1424   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1425   snat_main_per_thread_data_t *tsm;
1426   snat_session_t *s;
1427   u32 *locals = 0;
1428   uword *bitmap = 0;
1429   int i;
1430
1431   if (!sm->endpoint_dependent)
1432     return VNET_API_ERROR_FEATURE_DISABLED;
1433
1434   init_nat_k (&kv, e_addr, e_port, 0, proto);
1435   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1436     m = pool_elt_at_index (sm->static_mappings, value.value);
1437
1438   if (!m)
1439     return VNET_API_ERROR_NO_SUCH_ENTRY;
1440
1441   if (!is_lb_static_mapping (m))
1442     return VNET_API_ERROR_INVALID_VALUE;
1443
1444   /* *INDENT-OFF* */
1445   pool_foreach (local, m->locals)
1446    {
1447     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1448         (local->vrf_id == vrf_id))
1449       {
1450         match_local = local;
1451         break;
1452       }
1453   }
1454   /* *INDENT-ON* */
1455
1456   if (is_add)
1457     {
1458       if (match_local)
1459         return VNET_API_ERROR_VALUE_EXIST;
1460
1461       pool_get (m->locals, local);
1462       clib_memset (local, 0, sizeof (*local));
1463       local->addr.as_u32 = l_addr.as_u32;
1464       local->port = l_port;
1465       local->probability = probability;
1466       local->vrf_id = vrf_id;
1467       local->fib_index =
1468         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1469                                            sm->fib_src_low);
1470
1471       if (!is_out2in_only_static_mapping (m))
1472         {
1473           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto, 0,
1474                        m - sm->static_mappings);
1475           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1476             nat_elog_err ("static_mapping_by_local key add failed");
1477         }
1478     }
1479   else
1480     {
1481       if (!match_local)
1482         return VNET_API_ERROR_NO_SUCH_ENTRY;
1483
1484       if (pool_elts (m->locals) < 3)
1485         return VNET_API_ERROR_UNSPECIFIED;
1486
1487       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1488                         sm->fib_src_low);
1489
1490       if (!is_out2in_only_static_mapping (m))
1491         {
1492           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1493           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1494             nat_elog_err ("static_mapping_by_local key del failed");
1495         }
1496
1497       if (sm->num_workers > 1)
1498         {
1499           ip4_header_t ip = {
1500             .src_address = local->addr,
1501           };
1502           tsm = vec_elt_at_index (sm->per_thread_data,
1503                                   sm->worker_in2out_cb (&ip, m->fib_index,
1504                                                         0));
1505         }
1506       else
1507         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1508
1509       /* Delete sessions */
1510       /* *INDENT-OFF* */
1511       pool_foreach (s, tsm->sessions) {
1512         if (!(is_lb_session (s)))
1513           continue;
1514
1515         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1516             s->in2out.port != match_local->port)
1517           continue;
1518
1519         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1520         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1521       }
1522       /* *INDENT-ON* */
1523
1524       pool_put (m->locals, match_local);
1525     }
1526
1527   vec_free (m->workers);
1528
1529   /* *INDENT-OFF* */
1530   pool_foreach (local, m->locals)
1531    {
1532     vec_add1 (locals, local - m->locals);
1533     if (sm->num_workers > 1)
1534       {
1535         ip4_header_t ip;
1536         ip.src_address.as_u32 = local->addr.as_u32,
1537         bitmap = clib_bitmap_set (bitmap,
1538                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1539                                   1);
1540       }
1541   }
1542   /* *INDENT-ON* */
1543
1544   ASSERT (vec_len (locals) > 1);
1545
1546   local = pool_elt_at_index (m->locals, locals[0]);
1547   local->prefix = local->probability;
1548   for (i = 1; i < vec_len (locals); i++)
1549     {
1550       local = pool_elt_at_index (m->locals, locals[i]);
1551       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1552       local->prefix = local->probability + prev_local->prefix;
1553     }
1554
1555   /* Assign workers */
1556   if (sm->num_workers > 1)
1557     {
1558       /* *INDENT-OFF* */
1559       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1560       /* *INDENT-ON* */
1561     }
1562
1563   return 0;
1564 }
1565
1566 int
1567 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1568                   u8 twice_nat)
1569 {
1570   snat_address_t *a = 0;
1571   snat_session_t *ses;
1572   u32 *ses_to_be_removed = 0, *ses_index;
1573   snat_main_per_thread_data_t *tsm;
1574   snat_static_mapping_t *m;
1575   snat_interface_t *interface;
1576   int i;
1577   snat_address_t *addresses =
1578     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1579
1580   /* Find SNAT address */
1581   for (i = 0; i < vec_len (addresses); i++)
1582     {
1583       if (addresses[i].addr.as_u32 == addr.as_u32)
1584         {
1585           a = addresses + i;
1586           break;
1587         }
1588     }
1589   if (!a)
1590     {
1591       nat_log_err ("no such address");
1592       return VNET_API_ERROR_NO_SUCH_ENTRY;
1593     }
1594
1595   if (delete_sm)
1596     {
1597       ip4_address_t pool_addr = { 0 };
1598       /* *INDENT-OFF* */
1599       pool_foreach (m, sm->static_mappings)
1600        {
1601           if (m->external_addr.as_u32 == addr.as_u32)
1602             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1603                                             m->local_port, m->external_port,
1604                                             m->vrf_id,
1605                                             is_addr_only_static_mapping(m), ~0,
1606                                             m->proto, 0 /* is_add */,
1607                                             m->twice_nat,
1608                                             is_out2in_only_static_mapping(m),
1609                                             m->tag,
1610                                             is_identity_static_mapping(m),
1611                                             pool_addr, 0);
1612       }
1613       /* *INDENT-ON* */
1614     }
1615   else
1616     {
1617       /* Check if address is used in some static mapping */
1618       if (is_snat_address_used_in_static_mapping (sm, addr))
1619         {
1620           nat_log_err ("address used in static mapping");
1621           return VNET_API_ERROR_UNSPECIFIED;
1622         }
1623     }
1624
1625   if (a->fib_index != ~0)
1626     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1627
1628   /* Delete sessions using address */
1629   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1630     {
1631       vec_foreach (tsm, sm->per_thread_data)
1632       {
1633         /* *INDENT-OFF* */
1634         pool_foreach (ses, tsm->sessions)  {
1635           if (ses->out2in.addr.as_u32 == addr.as_u32)
1636             {
1637               nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1638               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1639             }
1640         }
1641         /* *INDENT-ON* */
1642
1643         if (sm->endpoint_dependent)
1644           {
1645             vec_foreach (ses_index, ses_to_be_removed)
1646             {
1647               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1648               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1649             }
1650           }
1651         else
1652           {
1653             vec_foreach (ses_index, ses_to_be_removed)
1654             {
1655               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1656               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1657             }
1658           }
1659
1660         vec_free (ses_to_be_removed);
1661       }
1662     }
1663
1664 #define _(N, i, n, s) \
1665   vec_free (a->busy_##n##_ports_per_thread);
1666   foreach_nat_protocol
1667 #undef _
1668
1669     if (twice_nat)
1670   {
1671     vec_del1 (sm->twice_nat_addresses, i);
1672     return 0;
1673   }
1674   else vec_del1 (sm->addresses, i);
1675
1676   /* Delete external address from FIB */
1677   pool_foreach (interface, sm->interfaces)
1678     {
1679       if (nat_interface_is_inside (interface) || sm->out2in_dpo)
1680         continue;
1681
1682       snat_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
1683       break;
1684     }
1685
1686   pool_foreach (interface, sm->output_feature_interfaces)
1687    {
1688     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1689       continue;
1690
1691     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1692     break;
1693   }
1694
1695   return 0;
1696 }
1697
1698 static void
1699 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1700 {
1701 #define _(x)                                                                  \
1702   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1703                                 sw_if_index);                                 \
1704   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1705   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1706                                 sw_if_index);                                 \
1707   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1708   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1709                                 sw_if_index);                                 \
1710   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1711   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1712                                 sw_if_index);                                 \
1713   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1714   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1715                                 sw_if_index);                                 \
1716   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1717   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1718                                 sw_if_index);                                 \
1719   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1720   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1721                                 sw_if_index);                                 \
1722   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1723   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1724                                 sw_if_index);                                 \
1725   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1726   foreach_nat_counter;
1727 #undef _
1728   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1729   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1730 }
1731
1732 void
1733 expire_per_vrf_sessions (u32 fib_index)
1734 {
1735   per_vrf_sessions_t *per_vrf_sessions;
1736   snat_main_per_thread_data_t *tsm;
1737   snat_main_t *sm = &snat_main;
1738
1739   /* *INDENT-OFF* */
1740   vec_foreach (tsm, sm->per_thread_data)
1741     {
1742       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1743         {
1744           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1745               (per_vrf_sessions->tx_fib_index == fib_index))
1746             {
1747               per_vrf_sessions->expired = 1;
1748             }
1749         }
1750     }
1751   /* *INDENT-ON* */
1752 }
1753
1754 void
1755 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1756 {
1757   snat_main_t *sm = &snat_main;
1758   nat_fib_t *fib;
1759
1760   // we don't care if it is outside/inside fib
1761   // we just care about their ref_count
1762   // if it reaches 0 sessions should expire
1763   // because the fib isn't valid for NAT anymore
1764
1765   vec_foreach (fib, sm->fibs)
1766   {
1767     if (fib->fib_index == fib_index)
1768       {
1769         if (is_del)
1770           {
1771             fib->ref_count--;
1772             if (!fib->ref_count)
1773               {
1774                 vec_del1 (sm->fibs, fib - sm->fibs);
1775                 expire_per_vrf_sessions (fib_index);
1776               }
1777             return;
1778           }
1779         else
1780           fib->ref_count++;
1781       }
1782   }
1783   if (!is_del)
1784     {
1785       vec_add2 (sm->fibs, fib, 1);
1786       fib->ref_count = 1;
1787       fib->fib_index = fib_index;
1788     }
1789 }
1790
1791 int
1792 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1793 {
1794   snat_main_t *sm = &snat_main;
1795   snat_interface_t *i;
1796   const char *feature_name, *del_feature_name;
1797   snat_address_t *ap;
1798   snat_static_mapping_t *m;
1799   nat_outside_fib_t *outside_fib;
1800   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1801                                                        sw_if_index);
1802
1803   if (!sm->enabled)
1804     {
1805       nat_log_err ("nat44 is disabled");
1806       return VNET_API_ERROR_UNSUPPORTED;
1807     }
1808
1809   if (sm->out2in_dpo && !is_inside)
1810     {
1811       nat_log_err ("error unsupported");
1812       return VNET_API_ERROR_UNSUPPORTED;
1813     }
1814
1815   /* *INDENT-OFF* */
1816   pool_foreach (i, sm->output_feature_interfaces)
1817    {
1818     if (i->sw_if_index == sw_if_index)
1819       {
1820         nat_log_err ("error interface already configured");
1821         return VNET_API_ERROR_VALUE_EXIST;
1822       }
1823   }
1824   /* *INDENT-ON* */
1825
1826   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1827     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1828   else
1829     {
1830       if (sm->num_workers > 1)
1831         feature_name =
1832           is_inside ? "nat44-in2out-worker-handoff" :
1833           "nat44-out2in-worker-handoff";
1834       else if (sm->endpoint_dependent)
1835         {
1836           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1837         }
1838       else
1839         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1840     }
1841
1842   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
1843     sm->fq_in2out_index =
1844       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1845
1846   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1847     sm->fq_out2in_index =
1848       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1849
1850   if (sm->endpoint_dependent)
1851     update_per_vrf_sessions_vec (fib_index, is_del);
1852
1853   if (!is_inside)
1854     {
1855       /* *INDENT-OFF* */
1856       vec_foreach (outside_fib, sm->outside_fibs)
1857         {
1858           if (outside_fib->fib_index == fib_index)
1859             {
1860               if (is_del)
1861                 {
1862                   outside_fib->refcount--;
1863                   if (!outside_fib->refcount)
1864                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1865                 }
1866               else
1867                 outside_fib->refcount++;
1868               goto feature_set;
1869             }
1870         }
1871       /* *INDENT-ON* */
1872       if (!is_del)
1873         {
1874           vec_add2 (sm->outside_fibs, outside_fib, 1);
1875           outside_fib->refcount = 1;
1876           outside_fib->fib_index = fib_index;
1877         }
1878     }
1879
1880 feature_set:
1881   /* *INDENT-OFF* */
1882   pool_foreach (i, sm->interfaces)
1883    {
1884     if (i->sw_if_index == sw_if_index)
1885       {
1886         if (is_del)
1887           {
1888             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1889               {
1890                 if (is_inside)
1891                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1892                 else
1893                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1894
1895                 if (sm->num_workers > 1)
1896                   {
1897                     del_feature_name = "nat44-handoff-classify";
1898                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1899                                                  "nat44-out2in-worker-handoff";
1900                   }
1901                 else if (sm->endpoint_dependent)
1902                   {
1903                     del_feature_name = "nat44-ed-classify";
1904                     feature_name = !is_inside ?  "nat-pre-in2out" :
1905                                                  "nat-pre-out2in";
1906                   }
1907                 else
1908                   {
1909                     del_feature_name = "nat44-classify";
1910                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1911                   }
1912
1913                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1914                 if (rv)
1915                   return rv;
1916                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1917                                              sw_if_index, 0, 0, 0);
1918                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1919                                              sw_if_index, 1, 0, 0);
1920                 if (!is_inside)
1921                   {
1922                     if (!sm->endpoint_dependent)
1923                       vnet_feature_enable_disable ("ip4-local",
1924                                                    "nat44-hairpinning",
1925                                                    sw_if_index, 1, 0, 0);
1926                   }
1927               }
1928             else
1929               {
1930                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1931                 if (rv)
1932                   return rv;
1933                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1934                                              sw_if_index, 0, 0, 0);
1935                 pool_put (sm->interfaces, i);
1936                 if (is_inside)
1937                   {
1938                     if (!sm->endpoint_dependent)
1939                       vnet_feature_enable_disable ("ip4-local",
1940                                                    "nat44-hairpinning",
1941                                                    sw_if_index, 0, 0, 0);
1942                   }
1943               }
1944           }
1945         else
1946           {
1947             if ((nat_interface_is_inside (i) && is_inside) ||
1948                 (nat_interface_is_outside (i) && !is_inside))
1949               return 0;
1950
1951             if (sm->num_workers > 1)
1952               {
1953                 del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1954                                                 "nat44-out2in-worker-handoff";
1955                 feature_name = "nat44-handoff-classify";
1956               }
1957             else if (sm->endpoint_dependent)
1958               {
1959                 del_feature_name =
1960                   !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1961
1962                 feature_name = "nat44-ed-classify";
1963               }
1964             else
1965               {
1966                 del_feature_name =
1967                   !is_inside ? "nat44-in2out" : "nat44-out2in";
1968                 feature_name = "nat44-classify";
1969               }
1970
1971             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1972             if (rv)
1973               return rv;
1974             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1975                                          sw_if_index, 0, 0, 0);
1976             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1977                                          sw_if_index, 1, 0, 0);
1978             if (!is_inside)
1979               {
1980                 if (!sm->endpoint_dependent)
1981                   vnet_feature_enable_disable (
1982                     "ip4-local", "nat44-hairpinning", sw_if_index, 0, 0, 0);
1983               }
1984             goto set_flags;
1985           }
1986
1987         goto fib;
1988       }
1989   }
1990   /* *INDENT-ON* */
1991
1992   if (is_del)
1993     {
1994       nat_log_err ("error interface couldn't be found");
1995       return VNET_API_ERROR_NO_SUCH_ENTRY;
1996     }
1997
1998   pool_get (sm->interfaces, i);
1999   i->sw_if_index = sw_if_index;
2000   i->flags = 0;
2001   nat_validate_counters (sm, sw_if_index);
2002
2003   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2004                                0);
2005
2006   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2007   if (rv)
2008     return rv;
2009
2010   if (is_inside && !sm->out2in_dpo)
2011     {
2012       if (!sm->endpoint_dependent)
2013         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2014                                      sw_if_index, 1, 0, 0);
2015     }
2016
2017 set_flags:
2018   if (is_inside)
2019     {
2020       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2021       return 0;
2022     }
2023   else
2024     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2025
2026   /* Add/delete external addresses to FIB */
2027 fib:
2028   /* *INDENT-OFF* */
2029   vec_foreach (ap, sm->addresses)
2030     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2031
2032   pool_foreach (m, sm->static_mappings)
2033    {
2034     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2035       continue;
2036
2037     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2038   }
2039   /* *INDENT-ON* */
2040
2041   return 0;
2042 }
2043
2044 int
2045 snat_interface_add_del_output_feature (u32 sw_if_index,
2046                                        u8 is_inside, int is_del)
2047 {
2048   snat_main_t *sm = &snat_main;
2049   snat_interface_t *i;
2050   snat_address_t *ap;
2051   snat_static_mapping_t *m;
2052   nat_outside_fib_t *outside_fib;
2053   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2054                                                        sw_if_index);
2055
2056   if (!sm->enabled)
2057     {
2058       nat_log_err ("nat44 is disabled");
2059       return VNET_API_ERROR_UNSUPPORTED;
2060     }
2061
2062   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2063     {
2064       nat_log_err ("error unsupported");
2065       return VNET_API_ERROR_UNSUPPORTED;
2066     }
2067
2068   /* *INDENT-OFF* */
2069   pool_foreach (i, sm->interfaces)
2070    {
2071     if (i->sw_if_index == sw_if_index)
2072       {
2073         nat_log_err ("error interface already configured");
2074         return VNET_API_ERROR_VALUE_EXIST;
2075       }
2076   }
2077   /* *INDENT-ON* */
2078
2079   if (sm->endpoint_dependent)
2080     update_per_vrf_sessions_vec (fib_index, is_del);
2081
2082   if (!is_inside)
2083     {
2084       /* *INDENT-OFF* */
2085       vec_foreach (outside_fib, sm->outside_fibs)
2086         {
2087           if (outside_fib->fib_index == fib_index)
2088             {
2089               if (is_del)
2090                 {
2091                   outside_fib->refcount--;
2092                   if (!outside_fib->refcount)
2093                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2094                 }
2095               else
2096                 outside_fib->refcount++;
2097               goto feature_set;
2098             }
2099         }
2100       /* *INDENT-ON* */
2101       if (!is_del)
2102         {
2103           vec_add2 (sm->outside_fibs, outside_fib, 1);
2104           outside_fib->refcount = 1;
2105           outside_fib->fib_index = fib_index;
2106         }
2107     }
2108
2109 feature_set:
2110   if (is_inside)
2111     {
2112       if (sm->endpoint_dependent)
2113         {
2114           int rv =
2115             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2116           if (rv)
2117             return rv;
2118           rv =
2119             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2120                                                             !is_del);
2121           if (rv)
2122             return rv;
2123         }
2124       else
2125         {
2126           int rv =
2127             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2128           if (rv)
2129             return rv;
2130           rv =
2131             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2132                                                             !is_del);
2133           if (rv)
2134             return rv;
2135           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2136                                        sw_if_index, !is_del, 0, 0);
2137           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2138                                        sw_if_index, !is_del, 0, 0);
2139         }
2140       goto fq;
2141     }
2142
2143   if (sm->num_workers > 1)
2144     {
2145       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2146       if (rv)
2147         return rv;
2148       rv =
2149         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2150       if (rv)
2151         return rv;
2152       vnet_feature_enable_disable ("ip4-unicast",
2153                                    "nat44-out2in-worker-handoff",
2154                                    sw_if_index, !is_del, 0, 0);
2155       vnet_feature_enable_disable ("ip4-output",
2156                                    "nat44-in2out-output-worker-handoff",
2157                                    sw_if_index, !is_del, 0, 0);
2158     }
2159   else
2160     {
2161       if (sm->endpoint_dependent)
2162         {
2163           int rv =
2164             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2165           if (rv)
2166             return rv;
2167           rv =
2168             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2169                                                             !is_del);
2170           if (rv)
2171             return rv;
2172           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2173                                        sw_if_index, !is_del, 0, 0);
2174           vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2175                                        sw_if_index, !is_del, 0, 0);
2176         }
2177       else
2178         {
2179           int rv =
2180             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2181           if (rv)
2182             return rv;
2183           rv =
2184             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2185                                                             !is_del);
2186           if (rv)
2187             return rv;
2188           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2189                                        sw_if_index, !is_del, 0, 0);
2190           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2191                                        sw_if_index, !is_del, 0, 0);
2192         }
2193     }
2194
2195 fq:
2196   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2197     sm->fq_in2out_output_index =
2198       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2199
2200   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2201     sm->fq_out2in_index =
2202       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2203
2204   /* *INDENT-OFF* */
2205   pool_foreach (i, sm->output_feature_interfaces)
2206    {
2207     if (i->sw_if_index == sw_if_index)
2208       {
2209         if (is_del)
2210           pool_put (sm->output_feature_interfaces, i);
2211         else
2212           return VNET_API_ERROR_VALUE_EXIST;
2213
2214         goto fib;
2215       }
2216   }
2217   /* *INDENT-ON* */
2218
2219   if (is_del)
2220     {
2221       nat_log_err ("error interface couldn't be found");
2222       return VNET_API_ERROR_NO_SUCH_ENTRY;
2223     }
2224
2225   pool_get (sm->output_feature_interfaces, i);
2226   i->sw_if_index = sw_if_index;
2227   i->flags = 0;
2228   nat_validate_counters (sm, sw_if_index);
2229   if (is_inside)
2230     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2231   else
2232     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2233
2234   /* Add/delete external addresses to FIB */
2235 fib:
2236   if (is_inside)
2237     return 0;
2238
2239   /* *INDENT-OFF* */
2240   vec_foreach (ap, sm->addresses)
2241     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2242
2243   pool_foreach (m, sm->static_mappings)
2244    {
2245     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2246       continue;
2247
2248     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2249   }
2250   /* *INDENT-ON* */
2251
2252   return 0;
2253 }
2254
2255 int
2256 snat_set_workers (uword * bitmap)
2257 {
2258   snat_main_t *sm = &snat_main;
2259   int i, j = 0;
2260
2261   if (sm->num_workers < 2)
2262     return VNET_API_ERROR_FEATURE_DISABLED;
2263
2264   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2265     return VNET_API_ERROR_INVALID_WORKER;
2266
2267   vec_free (sm->workers);
2268   /* *INDENT-OFF* */
2269   clib_bitmap_foreach (i, bitmap)
2270     {
2271       vec_add1(sm->workers, i);
2272       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2273       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2274       j++;
2275     }
2276   /* *INDENT-ON* */
2277
2278   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2279
2280   return 0;
2281 }
2282
2283 static void
2284 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2285                          u32 sw_if_index, u32 new_fib_index,
2286                          u32 old_fib_index)
2287 {
2288   snat_main_t *sm = &snat_main;
2289   nat_outside_fib_t *outside_fib;
2290   snat_interface_t *i;
2291   u8 is_add = 1;
2292   u8 match = 0;
2293
2294   if (!sm->enabled || (new_fib_index == old_fib_index)
2295       || (!vec_len (sm->outside_fibs)))
2296     {
2297       return;
2298     }
2299
2300   /* *INDENT-OFF* */
2301   pool_foreach (i, sm->interfaces)
2302      {
2303       if (i->sw_if_index == sw_if_index)
2304         {
2305           if (!(nat_interface_is_outside (i)))
2306             return;
2307           match = 1;
2308         }
2309     }
2310
2311   pool_foreach (i, sm->output_feature_interfaces)
2312      {
2313       if (i->sw_if_index == sw_if_index)
2314         {
2315           if (!(nat_interface_is_outside (i)))
2316             return;
2317           match = 1;
2318         }
2319     }
2320   /* *INDENT-ON* */
2321
2322   if (!match)
2323     return;
2324
2325   vec_foreach (outside_fib, sm->outside_fibs)
2326   {
2327     if (outside_fib->fib_index == old_fib_index)
2328       {
2329         outside_fib->refcount--;
2330         if (!outside_fib->refcount)
2331           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2332         break;
2333       }
2334   }
2335
2336   vec_foreach (outside_fib, sm->outside_fibs)
2337   {
2338     if (outside_fib->fib_index == new_fib_index)
2339       {
2340         outside_fib->refcount++;
2341         is_add = 0;
2342         break;
2343       }
2344   }
2345
2346   if (is_add)
2347     {
2348       vec_add2 (sm->outside_fibs, outside_fib, 1);
2349       outside_fib->refcount = 1;
2350       outside_fib->fib_index = new_fib_index;
2351     }
2352 }
2353
2354 static void
2355 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2356                          u32 sw_if_index, u32 new_fib_index,
2357                          u32 old_fib_index);
2358
2359 static void
2360 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2361                                        uword opaque,
2362                                        u32 sw_if_index,
2363                                        ip4_address_t * address,
2364                                        u32 address_length,
2365                                        u32 if_address_index, u32 is_delete);
2366
2367 static void
2368 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2369                                  uword opaque,
2370                                  u32 sw_if_index,
2371                                  ip4_address_t * address,
2372                                  u32 address_length,
2373                                  u32 if_address_index, u32 is_delete);
2374
2375 void
2376 test_key_calc_split ()
2377 {
2378   ip4_address_t l_addr;
2379   l_addr.as_u8[0] = 1;
2380   l_addr.as_u8[1] = 1;
2381   l_addr.as_u8[2] = 1;
2382   l_addr.as_u8[3] = 1;
2383   ip4_address_t r_addr;
2384   r_addr.as_u8[0] = 2;
2385   r_addr.as_u8[1] = 2;
2386   r_addr.as_u8[2] = 2;
2387   r_addr.as_u8[3] = 2;
2388   u16 l_port = 40001;
2389   u16 r_port = 40301;
2390   u8 proto = 9;
2391   u32 fib_index = 9000001;
2392   u32 thread_index = 3000000001;
2393   u32 session_index = 3000000221;
2394   clib_bihash_kv_16_8_t kv;
2395   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2396               thread_index, session_index);
2397   ip4_address_t l_addr2;
2398   ip4_address_t r_addr2;
2399   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2400   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2401   u16 l_port2 = 0;
2402   u16 r_port2 = 0;
2403   u8 proto2 = 0;
2404   u32 fib_index2 = 0;
2405   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2406                &r_port2);
2407   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2408   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2409   ASSERT (l_port == l_port2);
2410   ASSERT (r_port == r_port2);
2411   ASSERT (proto == proto2);
2412   ASSERT (fib_index == fib_index2);
2413   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2414   ASSERT (session_index == ed_value_get_session_index (&kv));
2415
2416   fib_index = 7001;
2417   proto = 5;
2418   nat_protocol_t proto3 = ~0;
2419   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2420   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2421   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2422   ASSERT (l_port == l_port2);
2423   ASSERT (proto == proto3);
2424   ASSERT (fib_index == fib_index2);
2425 }
2426
2427 static clib_error_t *
2428 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2429 {
2430   snat_main_t *sm = &snat_main;
2431   u32 fib_index;
2432
2433   if (sm->endpoint_dependent)
2434     {
2435       // TODO: consider removing all NAT interfaces
2436       if (!is_add)
2437         {
2438           fib_index = ip4_fib_index_from_table_id (table_id);
2439           if (fib_index != ~0)
2440             expire_per_vrf_sessions (fib_index);
2441         }
2442     }
2443   return 0;
2444 }
2445
2446 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2447
2448 void
2449 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2450 {
2451   vlib_node_t *node;
2452
2453   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2454   sm->ei_out2in_node_index = node->index;
2455   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2456   sm->ei_in2out_node_index = node->index;
2457   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2458   sm->ei_in2out_output_node_index = node->index;
2459
2460   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2461   sm->ed_out2in_node_index = node->index;
2462   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2463   sm->ed_in2out_node_index = node->index;
2464   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2465   sm->ed_in2out_output_node_index = node->index;
2466
2467   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2468   sm->error_node_index = node->index;
2469   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2470   sm->pre_in2out_node_index = node->index;
2471   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2472   sm->pre_out2in_node_index = node->index;
2473   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2474   sm->pre_in2out_node_index = node->index;
2475   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2476   sm->pre_out2in_node_index = node->index;
2477   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2478   sm->in2out_fast_node_index = node->index;
2479   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2480   sm->in2out_slowpath_node_index = node->index;
2481   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2482   sm->in2out_slowpath_output_node_index = node->index;
2483   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2484   sm->ed_in2out_slowpath_node_index = node->index;
2485   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2486   sm->out2in_fast_node_index = node->index;
2487   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2488   sm->ed_out2in_slowpath_node_index = node->index;
2489   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2490   sm->hairpinning_node_index = node->index;
2491   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2492   sm->hairpin_dst_node_index = node->index;
2493   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2494   sm->hairpin_src_node_index = node->index;
2495 }
2496
2497 #define nat_init_simple_counter(c, n, sn) \
2498 do                                        \
2499   {                                       \
2500     c.name = n;                           \
2501     c.stat_segment_name = sn;             \
2502     vlib_validate_simple_counter (&c, 0); \
2503     vlib_zero_simple_counter (&c, 0);     \
2504   } while (0);
2505
2506 static clib_error_t *
2507 nat_init (vlib_main_t * vm)
2508 {
2509   snat_main_t *sm = &snat_main;
2510   vlib_thread_main_t *tm = vlib_get_thread_main ();
2511   vlib_thread_registration_t *tr;
2512   ip4_add_del_interface_address_callback_t cbi = { 0 };
2513   ip4_table_bind_callback_t cbt = { 0 };
2514   u32 i, num_threads = 0;
2515   uword *p, *bitmap = 0;
2516
2517   clib_memset (sm, 0, sizeof (*sm));
2518
2519   // required
2520   sm->vnet_main = vnet_get_main ();
2521   // convenience
2522   sm->ip4_main = &ip4_main;
2523   sm->api_main = vlibapi_get_main ();
2524   sm->ip4_lookup_main = &ip4_main.lookup_main;
2525
2526   // frame queue indices used for handoff
2527   sm->fq_out2in_index = ~0;
2528   sm->fq_in2out_index = ~0;
2529   sm->fq_in2out_output_index = ~0;
2530
2531   sm->log_level = SNAT_LOG_ERROR;
2532
2533   nat44_set_node_indexes (sm, vm);
2534   sm->log_class = vlib_log_register_class ("nat", 0);
2535   nat_ipfix_logging_init (vm);
2536
2537   nat_init_simple_counter (sm->total_users, "total-users",
2538                            "/nat44/total-users");
2539   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2540                            "/nat44/total-sessions");
2541   nat_init_simple_counter (sm->user_limit_reached, "user-limit-reached",
2542                            "/nat44/user-limit-reached");
2543
2544 #define _(x)                                            \
2545   sm->counters.fastpath.in2out.x.name = #x;             \
2546   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2547       "/nat44/in2out/fastpath/" #x;                     \
2548   sm->counters.slowpath.in2out.x.name = #x;             \
2549   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2550       "/nat44/in2out/slowpath/" #x;                     \
2551   sm->counters.fastpath.out2in.x.name = #x;             \
2552   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2553       "/nat44/out2in/fastpath/" #x;                     \
2554   sm->counters.slowpath.out2in.x.name = #x;             \
2555   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2556       "/nat44/out2in/slowpath/" #x;                     \
2557   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2558   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2559       "/nat44/ed/in2out/fastpath/" #x;                  \
2560   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2561   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2562       "/nat44/ed/in2out/slowpath/" #x;                  \
2563   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2564   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2565       "/nat44/ed/out2in/fastpath/" #x;                  \
2566   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2567   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2568       "/nat44/ed/out2in/slowpath/" #x;
2569   foreach_nat_counter;
2570 #undef _
2571   sm->counters.hairpinning.name = "hairpinning";
2572   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2573
2574   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2575   if (p)
2576     {
2577       tr = (vlib_thread_registration_t *) p[0];
2578       if (tr)
2579         {
2580           sm->num_workers = tr->count;
2581           sm->first_worker_index = tr->first_index;
2582         }
2583     }
2584   num_threads = tm->n_vlib_mains - 1;
2585   sm->port_per_thread = 0xffff - 1024;
2586   vec_validate (sm->per_thread_data, num_threads);
2587
2588   /* Use all available workers by default */
2589   if (sm->num_workers > 1)
2590     {
2591
2592       for (i = 0; i < sm->num_workers; i++)
2593         bitmap = clib_bitmap_set (bitmap, i, 1);
2594       snat_set_workers (bitmap);
2595       clib_bitmap_free (bitmap);
2596     }
2597   else
2598     sm->per_thread_data[0].snat_thread_index = 0;
2599
2600   /* callbacks to call when interface address changes. */
2601   cbi.function = snat_ip4_add_del_interface_address_cb;
2602   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2603   cbi.function = nat_ip4_add_del_addr_only_sm_cb;
2604   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2605
2606   /* callbacks to call when interface to table biding changes */
2607   cbt.function = snat_update_outside_fib;
2608   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2609
2610   sm->fib_src_low =
2611     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2612                          FIB_SOURCE_BH_SIMPLE);
2613   sm->fib_src_hi =
2614     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2615                          FIB_SOURCE_BH_SIMPLE);
2616
2617   /* used only by out2in-dpo feature */
2618   nat_dpo_module_init ();
2619
2620   nat_affinity_init (vm);
2621   nat_ha_init (vm, sm->num_workers, num_threads);
2622
2623   test_key_calc_split ();
2624   return nat44_api_hookup (vm);
2625 }
2626
2627 VLIB_INIT_FUNCTION (nat_init);
2628
2629 static int
2630 nat44_ed_plugin_enable (nat44_config_t c)
2631 {
2632   snat_main_t *sm = &snat_main;
2633
2634   if (c.static_mapping_only && !c.connection_tracking)
2635     {
2636       nat_log_err ("unsupported combination of configuration");
2637       return 1;
2638     }
2639
2640   // nat44 feature configuration
2641   sm->endpoint_dependent = c.endpoint_dependent;
2642   sm->static_mapping_only = c.static_mapping_only;
2643   sm->static_mapping_connection_tracking = c.connection_tracking;
2644
2645   // EI only feature (could break ED)
2646   sm->out2in_dpo = c.out2in_dpo;
2647
2648   sm->forwarding_enabled = 0;
2649   sm->mss_clamping = 0;
2650   sm->pat = (!c.static_mapping_only ||
2651              (c.static_mapping_only && c.connection_tracking));
2652
2653   if (!c.users)
2654     c.users = 1024;
2655
2656   // EI only feature (could break ED)
2657   sm->max_users_per_thread = c.users;
2658   sm->user_buckets = nat_calc_bihash_buckets (c.users);
2659
2660   if (!c.sessions)
2661     c.sessions = 10 * 1024;
2662
2663   sm->max_translations_per_thread = c.sessions;
2664   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2665
2666   // ED only feature
2667   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2668
2669   // EI only feature (could break ED)
2670   sm->max_translations_per_user
2671     = c.user_sessions ? c.user_sessions : sm->max_translations_per_thread;
2672
2673   sm->inside_vrf_id = c.inside_vrf;
2674   sm->inside_fib_index =
2675     fib_table_find_or_create_and_lock
2676     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2677
2678   sm->outside_vrf_id = c.outside_vrf;
2679   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2680     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2681
2682   sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
2683   sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2684
2685   sm->in2out_node_index = sm->ed_in2out_node_index;
2686   sm->out2in_node_index = sm->ed_out2in_node_index;
2687
2688   sm->in2out_output_node_index = sm->ed_in2out_output_node_index;
2689
2690   if (sm->pat)
2691     {
2692       sm->icmp_match_in2out_cb = NULL;
2693       sm->icmp_match_out2in_cb = NULL;
2694     }
2695   else
2696     {
2697       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2698       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2699     }
2700
2701   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets,
2702                     sm->user_buckets);
2703
2704   nat_affinity_enable ();
2705
2706   nat_reset_timeouts (&sm->timeouts);
2707
2708   // TODO: function for reset counters
2709   vlib_zero_simple_counter (&sm->total_users, 0);
2710   vlib_zero_simple_counter (&sm->total_sessions, 0);
2711   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2712
2713   sm->enabled = 1;
2714   sm->rconfig = c;
2715
2716   return 0;
2717 }
2718
2719 int
2720 nat44_plugin_enable (nat44_config_t c)
2721 {
2722   fail_if_enabled ();
2723
2724   // c.static_mapping_only + c.connection_tracking
2725   //  - supported in NAT EI & NAT ED
2726   // c.out2in_dpo, c.static_mapping_only
2727   //  - supported in NAT EI
2728
2729   if (c.endpoint_dependent)
2730     {
2731       if (c.out2in_dpo || c.users || c.user_sessions)
2732         {
2733           nat_log_err ("unsupported combination of configuration");
2734           return 1;
2735         }
2736       return nat44_ed_plugin_enable (c);
2737     }
2738
2739   // separation:
2740   // for now just copy variables
2741
2742   nat44_ei_config_t ei_c = {
2743     .inside_vrf = c.inside_vrf,
2744     .outside_vrf = c.outside_vrf,
2745     .users = c.users,
2746     .sessions = c.sessions,
2747     .user_sessions = c.user_sessions,
2748     .out2in_dpo = c.out2in_dpo,
2749     .static_mapping_only = c.static_mapping_only,
2750     .connection_tracking = c.connection_tracking,
2751   };
2752
2753   return nat44_ei_plugin_enable (ei_c);
2754 }
2755
2756 void
2757 nat44_addresses_free (snat_address_t ** addresses)
2758 {
2759   snat_address_t *ap;
2760   /* *INDENT-OFF* */
2761   vec_foreach (ap, *addresses)
2762     {
2763     #define _(N, i, n, s) \
2764       vec_free (ap->busy_##n##_ports_per_thread);
2765       foreach_nat_protocol
2766     #undef _
2767     }
2768   /* *INDENT-ON* */
2769   vec_free (*addresses);
2770   *addresses = 0;
2771 }
2772
2773 static int
2774 nat44_ed_plugin_disable ()
2775 {
2776   snat_main_t *sm = &snat_main;
2777   snat_interface_t *i, *vec;
2778   int error = 0;
2779
2780   // first unregister all nodes from interfaces
2781   vec = vec_dup (sm->interfaces);
2782   /* *INDENT-OFF* */
2783   vec_foreach (i, vec)
2784     {
2785       if (nat_interface_is_inside(i))
2786         error = snat_interface_add_del (i->sw_if_index, 1, 1);
2787       if (nat_interface_is_outside(i))
2788         error = snat_interface_add_del (i->sw_if_index, 0, 1);
2789
2790       if (error)
2791         {
2792           nat_log_err ("error occurred while removing interface %u",
2793                        i->sw_if_index);
2794         }
2795     }
2796   /* *INDENT-ON* */
2797   vec_free (vec);
2798   sm->interfaces = 0;
2799
2800   vec = vec_dup (sm->output_feature_interfaces);
2801   /* *INDENT-OFF* */
2802   vec_foreach (i, vec)
2803     {
2804       if (nat_interface_is_inside(i))
2805         error = snat_interface_add_del_output_feature (i->sw_if_index, 1, 1);
2806       if (nat_interface_is_outside(i))
2807         error = snat_interface_add_del_output_feature (i->sw_if_index, 0, 1);
2808
2809       if (error)
2810         {
2811           nat_log_err ("error occurred while removing interface %u",
2812                        i->sw_if_index);
2813         }
2814     }
2815   /* *INDENT-ON* */
2816   vec_free (vec);
2817   sm->output_feature_interfaces = 0;
2818
2819   vec_free (sm->max_translations_per_fib);
2820
2821   nat44_ed_db_free ();
2822
2823   nat44_addresses_free (&sm->addresses);
2824   nat44_addresses_free (&sm->twice_nat_addresses);
2825
2826
2827   vec_free (sm->to_resolve);
2828   vec_free (sm->auto_add_sw_if_indices);
2829   vec_free (sm->auto_add_sw_if_indices_twice_nat);
2830
2831   sm->to_resolve = 0;
2832   sm->auto_add_sw_if_indices = 0;
2833   sm->auto_add_sw_if_indices_twice_nat = 0;
2834
2835   sm->forwarding_enabled = 0;
2836
2837   sm->enabled = 0;
2838   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2839
2840   return 0;
2841 }
2842
2843 int
2844 nat44_plugin_disable ()
2845 {
2846   snat_main_t *sm = &snat_main;
2847
2848   fail_if_disabled ();
2849
2850   if (sm->endpoint_dependent)
2851     return nat44_ed_plugin_disable ();
2852   return nat44_ei_plugin_disable ();
2853 }
2854
2855 void
2856 snat_free_outside_address_and_port (snat_address_t *addresses,
2857                                     u32 thread_index, ip4_address_t *addr,
2858                                     u16 port, nat_protocol_t protocol)
2859 {
2860   snat_address_t *a;
2861   u32 address_index;
2862   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2863
2864   for (address_index = 0; address_index < vec_len (addresses);
2865        address_index++)
2866     {
2867       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2868         break;
2869     }
2870
2871   ASSERT (address_index < vec_len (addresses));
2872
2873   a = addresses + address_index;
2874
2875   switch (protocol)
2876     {
2877 #define _(N, i, n, s) \
2878     case NAT_PROTOCOL_##N: \
2879       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2880       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2881       a->busy_##n##_ports--; \
2882       a->busy_##n##_ports_per_thread[thread_index]--; \
2883       break;
2884       foreach_nat_protocol
2885 #undef _
2886     default:
2887       nat_elog_info ("unknown protocol");
2888       return;
2889     }
2890 }
2891
2892 int
2893 nat_set_outside_address_and_port (snat_address_t *addresses, u32 thread_index,
2894                                   ip4_address_t addr, u16 port,
2895                                   nat_protocol_t protocol)
2896 {
2897   snat_address_t *a = 0;
2898   u32 address_index;
2899   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2900
2901   for (address_index = 0; address_index < vec_len (addresses);
2902        address_index++)
2903     {
2904       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2905         continue;
2906
2907       a = addresses + address_index;
2908       switch (protocol)
2909         {
2910 #define _(N, j, n, s) \
2911         case NAT_PROTOCOL_##N: \
2912           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2913             return VNET_API_ERROR_INSTANCE_IN_USE; \
2914           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2915           a->busy_##n##_ports_per_thread[thread_index]++; \
2916           a->busy_##n##_ports++; \
2917           return 0;
2918           foreach_nat_protocol
2919 #undef _
2920         default:
2921           nat_elog_info ("unknown protocol");
2922           return 1;
2923         }
2924     }
2925
2926   return VNET_API_ERROR_NO_SUCH_ENTRY;
2927 }
2928
2929 int
2930 snat_static_mapping_match (snat_main_t * sm,
2931                            ip4_address_t match_addr,
2932                            u16 match_port,
2933                            u32 match_fib_index,
2934                            nat_protocol_t match_protocol,
2935                            ip4_address_t * mapping_addr,
2936                            u16 * mapping_port,
2937                            u32 * mapping_fib_index,
2938                            u8 by_external,
2939                            u8 * is_addr_only,
2940                            twice_nat_type_t * twice_nat,
2941                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2942                            u8 * is_identity_nat, snat_static_mapping_t ** out)
2943 {
2944   clib_bihash_kv_8_8_t kv, value;
2945   clib_bihash_8_8_t *mapping_hash;
2946   snat_static_mapping_t *m;
2947   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2948   nat44_lb_addr_port_t *local;
2949   u8 backend_index;
2950
2951   if (!by_external)
2952     {
2953       mapping_hash = &sm->static_mapping_by_local;
2954       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2955                   match_protocol);
2956       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2957         {
2958           /* Try address only mapping */
2959           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2960           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2961             return 1;
2962         }
2963     }
2964   else
2965     {
2966       mapping_hash = &sm->static_mapping_by_external;
2967       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2968       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2969         {
2970           /* Try address only mapping */
2971           init_nat_k (&kv, match_addr, 0, 0, 0);
2972           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2973             return 1;
2974         }
2975     }
2976
2977   m = pool_elt_at_index (sm->static_mappings, value.value);
2978
2979   if (by_external)
2980     {
2981       if (is_lb_static_mapping (m))
2982         {
2983           if (PREDICT_FALSE (lb != 0))
2984             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2985           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2986                                                           match_addr,
2987                                                           match_protocol,
2988                                                           match_port,
2989                                                           &backend_index))
2990             {
2991               local = pool_elt_at_index (m->locals, backend_index);
2992               *mapping_addr = local->addr;
2993               *mapping_port = local->port;
2994               *mapping_fib_index = local->fib_index;
2995               goto end;
2996             }
2997           // pick locals matching this worker
2998           if (PREDICT_FALSE (sm->num_workers > 1))
2999             {
3000               u32 thread_index = vlib_get_thread_index ();
3001               /* *INDENT-OFF* */
3002               pool_foreach_index (i, m->locals)
3003                {
3004                 local = pool_elt_at_index (m->locals, i);
3005
3006                 ip4_header_t ip = {
3007                   .src_address = local->addr,
3008                 };
3009
3010                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
3011                     thread_index)
3012                   {
3013                     vec_add1 (tmp, i);
3014                   }
3015               }
3016               /* *INDENT-ON* */
3017               ASSERT (vec_len (tmp) != 0);
3018             }
3019           else
3020             {
3021               /* *INDENT-OFF* */
3022               pool_foreach_index (i, m->locals)
3023                {
3024                 vec_add1 (tmp, i);
3025               }
3026               /* *INDENT-ON* */
3027             }
3028           hi = vec_len (tmp) - 1;
3029           local = pool_elt_at_index (m->locals, tmp[hi]);
3030           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
3031           while (lo < hi)
3032             {
3033               mid = ((hi - lo) >> 1) + lo;
3034               local = pool_elt_at_index (m->locals, tmp[mid]);
3035               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
3036             }
3037           local = pool_elt_at_index (m->locals, tmp[lo]);
3038           if (!(local->prefix >= rand))
3039             return 1;
3040           *mapping_addr = local->addr;
3041           *mapping_port = local->port;
3042           *mapping_fib_index = local->fib_index;
3043           if (m->affinity)
3044             {
3045               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
3046                                                 match_protocol, match_port,
3047                                                 tmp[lo], m->affinity,
3048                                                 m->affinity_per_service_list_head_index))
3049                 nat_elog_info ("create affinity record failed");
3050             }
3051           vec_free (tmp);
3052         }
3053       else
3054         {
3055           if (PREDICT_FALSE (lb != 0))
3056             *lb = NO_LB_NAT;
3057           *mapping_fib_index = m->fib_index;
3058           *mapping_addr = m->local_addr;
3059           /* Address only mapping doesn't change port */
3060           *mapping_port = is_addr_only_static_mapping (m) ? match_port
3061             : m->local_port;
3062         }
3063     }
3064   else
3065     {
3066       *mapping_addr = m->external_addr;
3067       /* Address only mapping doesn't change port */
3068       *mapping_port = is_addr_only_static_mapping (m) ? match_port
3069         : m->external_port;
3070       *mapping_fib_index = sm->outside_fib_index;
3071     }
3072
3073 end:
3074   if (PREDICT_FALSE (is_addr_only != 0))
3075     *is_addr_only = is_addr_only_static_mapping (m);
3076
3077   if (PREDICT_FALSE (twice_nat != 0))
3078     *twice_nat = m->twice_nat;
3079
3080   if (PREDICT_FALSE (is_identity_nat != 0))
3081     *is_identity_nat = is_identity_static_mapping (m);
3082
3083   if (out != 0)
3084     *out = m;
3085
3086   return 0;
3087 }
3088
3089 void
3090 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3091 {
3092   snat_main_t *sm = &snat_main;
3093   dpo_id_t dpo_v4 = DPO_INVALID;
3094   fib_prefix_t pfx = {
3095     .fp_proto = FIB_PROTOCOL_IP4,
3096     .fp_len = 32,
3097     .fp_addr.ip4.as_u32 = addr.as_u32,
3098   };
3099
3100   if (is_add)
3101     {
3102       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3103       fib_table_entry_special_dpo_add (0, &pfx, sm->fib_src_hi,
3104                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3105       dpo_reset (&dpo_v4);
3106     }
3107   else
3108     {
3109       fib_table_entry_special_remove (0, &pfx, sm->fib_src_hi);
3110     }
3111 }
3112
3113 static u32
3114 nat44_ed_get_worker_in2out_cb (ip4_header_t *ip, u32 rx_fib_index,
3115                                u8 is_output)
3116 {
3117   snat_main_t *sm = &snat_main;
3118   u32 next_worker_index = sm->first_worker_index;
3119   u32 hash;
3120
3121   clib_bihash_kv_16_8_t kv16, value16;
3122   snat_main_per_thread_data_t *tsm;
3123   udp_header_t *udp;
3124
3125   if (PREDICT_FALSE (is_output))
3126     {
3127       u32 fib_index = sm->outside_fib_index;
3128       nat_outside_fib_t *outside_fib;
3129       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3130       fib_prefix_t pfx = {
3131         .fp_proto = FIB_PROTOCOL_IP4,
3132         .fp_len = 32,
3133         .fp_addr = {
3134                     .ip4.as_u32 = ip->dst_address.as_u32,
3135                     }
3136         ,
3137       };
3138
3139       udp = ip4_next_header (ip);
3140
3141       switch (vec_len (sm->outside_fibs))
3142         {
3143         case 0:
3144           fib_index = sm->outside_fib_index;
3145           break;
3146         case 1:
3147           fib_index = sm->outside_fibs[0].fib_index;
3148           break;
3149         default:
3150             /* *INDENT-OFF* */
3151             vec_foreach (outside_fib, sm->outside_fibs)
3152               {
3153                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3154                 if (FIB_NODE_INDEX_INVALID != fei)
3155                   {
3156                     if (fib_entry_get_resolving_interface (fei) != ~0)
3157                       {
3158                         fib_index = outside_fib->fib_index;
3159                         break;
3160                       }
3161                   }
3162               }
3163             /* *INDENT-ON* */
3164           break;
3165         }
3166
3167       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3168                  udp->dst_port, fib_index, ip->protocol);
3169
3170       if (PREDICT_TRUE (
3171             !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3172         {
3173           tsm =
3174             vec_elt_at_index (sm->per_thread_data,
3175                               ed_value_get_thread_index (&value16));
3176           next_worker_index += tsm->thread_index;
3177
3178           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3179                                   next_worker_index, fib_index,
3180                                   clib_net_to_host_u32 (ip->
3181                                                         src_address.as_u32),
3182                                   clib_net_to_host_u32 (ip->
3183                                                         dst_address.as_u32));
3184
3185           return next_worker_index;
3186         }
3187     }
3188
3189   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3190     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3191
3192   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3193     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3194   else
3195     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3196
3197   if (PREDICT_TRUE (!is_output))
3198     {
3199       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3200                               next_worker_index, rx_fib_index,
3201                               clib_net_to_host_u32 (ip->src_address.as_u32),
3202                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3203     }
3204   else
3205     {
3206       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3207                               next_worker_index, rx_fib_index,
3208                               clib_net_to_host_u32 (ip->src_address.as_u32),
3209                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3210     }
3211
3212   return next_worker_index;
3213 }
3214
3215 static u32
3216 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3217                                u32 rx_fib_index, u8 is_output)
3218 {
3219   snat_main_t *sm = &snat_main;
3220   clib_bihash_kv_8_8_t kv, value;
3221   clib_bihash_kv_16_8_t kv16, value16;
3222   snat_main_per_thread_data_t *tsm;
3223
3224   u32 proto, next_worker_index = 0;
3225   udp_header_t *udp;
3226   u16 port;
3227   snat_static_mapping_t *m;
3228   u32 hash;
3229
3230   proto = ip_proto_to_nat_proto (ip->protocol);
3231
3232   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3233     {
3234       udp = ip4_next_header (ip);
3235
3236       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3237                  udp->src_port, rx_fib_index, ip->protocol);
3238
3239       if (PREDICT_TRUE (
3240             !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3241         {
3242           tsm =
3243             vec_elt_at_index (sm->per_thread_data,
3244                               ed_value_get_thread_index (&value16));
3245           vnet_buffer2 (b)->nat.cached_session_index =
3246             ed_value_get_session_index (&value16);
3247           next_worker_index = sm->first_worker_index + tsm->thread_index;
3248           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3249                                   next_worker_index, rx_fib_index,
3250                                   clib_net_to_host_u32 (ip->
3251                                                         src_address.as_u32),
3252                                   clib_net_to_host_u32 (ip->
3253                                                         dst_address.as_u32));
3254           return next_worker_index;
3255         }
3256     }
3257   else if (proto == NAT_PROTOCOL_ICMP)
3258     {
3259       ip4_address_t lookup_saddr, lookup_daddr;
3260       u16 lookup_sport, lookup_dport;
3261       u8 lookup_protocol;
3262       if (!nat_get_icmp_session_lookup_values (
3263             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3264             &lookup_protocol))
3265         {
3266           init_ed_k (&kv16, lookup_saddr, lookup_sport, lookup_daddr,
3267                      lookup_dport, rx_fib_index, lookup_protocol);
3268           if (PREDICT_TRUE (
3269                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3270             {
3271               tsm =
3272                 vec_elt_at_index (sm->per_thread_data,
3273                                   ed_value_get_thread_index (&value16));
3274               next_worker_index = sm->first_worker_index + tsm->thread_index;
3275               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3276                                       next_worker_index, rx_fib_index,
3277                                       clib_net_to_host_u32 (ip->
3278                                                             src_address.as_u32),
3279                                       clib_net_to_host_u32 (ip->
3280                                                             dst_address.as_u32));
3281               return next_worker_index;
3282             }
3283         }
3284     }
3285
3286   /* first try static mappings without port */
3287   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3288     {
3289       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3290       if (!clib_bihash_search_8_8
3291           (&sm->static_mapping_by_external, &kv, &value))
3292         {
3293           m = pool_elt_at_index (sm->static_mappings, value.value);
3294           next_worker_index = m->workers[0];
3295           goto done;
3296         }
3297     }
3298
3299   /* unknown protocol */
3300   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3301     {
3302       /* use current thread */
3303       next_worker_index = vlib_get_thread_index ();
3304       goto done;
3305     }
3306
3307   udp = ip4_next_header (ip);
3308   port = udp->dst_port;
3309
3310   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3311     {
3312       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3313       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3314       if (!icmp_type_is_error_message
3315           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3316         port = vnet_buffer (b)->ip.reass.l4_src_port;
3317       else
3318         {
3319           /* if error message, then it's not fragmented and we can access it */
3320           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3321           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3322           void *l4_header = ip4_next_header (inner_ip);
3323           switch (proto)
3324             {
3325             case NAT_PROTOCOL_ICMP:
3326               icmp = (icmp46_header_t *) l4_header;
3327               echo = (icmp_echo_header_t *) (icmp + 1);
3328               port = echo->identifier;
3329               break;
3330             case NAT_PROTOCOL_UDP:
3331             case NAT_PROTOCOL_TCP:
3332               port = ((tcp_udp_header_t *) l4_header)->src_port;
3333               break;
3334             default:
3335               next_worker_index = vlib_get_thread_index ();
3336               goto done;
3337             }
3338         }
3339     }
3340
3341   /* try static mappings with port */
3342   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3343     {
3344       init_nat_k (&kv, ip->dst_address, port, 0, proto);
3345       if (!clib_bihash_search_8_8
3346           (&sm->static_mapping_by_external, &kv, &value))
3347         {
3348           m = pool_elt_at_index (sm->static_mappings, value.value);
3349           if (!is_lb_static_mapping (m))
3350             {
3351               next_worker_index = m->workers[0];
3352               goto done;
3353             }
3354
3355           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3356             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3357
3358           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3359             next_worker_index =
3360               m->workers[hash & (_vec_len (m->workers) - 1)];
3361           else
3362             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3363           goto done;
3364         }
3365     }
3366
3367   /* worker by outside port */
3368   next_worker_index = sm->first_worker_index;
3369   next_worker_index +=
3370     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3371
3372 done:
3373   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3374                           clib_net_to_host_u32 (ip->src_address.as_u32),
3375                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3376   return next_worker_index;
3377 }
3378
3379 u32
3380 nat_calc_bihash_buckets (u32 n_elts)
3381 {
3382   n_elts = n_elts / 2.5;
3383   u64 lower_pow2 = 1;
3384   while (lower_pow2 * 2 < n_elts)
3385     {
3386       lower_pow2 = 2 * lower_pow2;
3387     }
3388   u64 upper_pow2 = 2 * lower_pow2;
3389   if ((upper_pow2 - n_elts) < (n_elts - lower_pow2))
3390     {
3391       if (upper_pow2 <= UINT32_MAX)
3392         {
3393           return upper_pow2;
3394         }
3395     }
3396   return lower_pow2;
3397 }
3398
3399 u32
3400 nat44_get_max_session_limit ()
3401 {
3402   snat_main_t *sm = &snat_main;
3403   u32 max_limit = 0, len = 0;
3404
3405   for (; len < vec_len (sm->max_translations_per_fib); len++)
3406     {
3407       if (max_limit < sm->max_translations_per_fib[len])
3408         max_limit = sm->max_translations_per_fib[len];
3409     }
3410   return max_limit;
3411 }
3412
3413 int
3414 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3415 {
3416   snat_main_t *sm = &snat_main;
3417   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3418   u32 len = vec_len (sm->max_translations_per_fib);
3419
3420   if (len <= fib_index)
3421     {
3422       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3423
3424       for (; len < vec_len (sm->max_translations_per_fib); len++)
3425         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3426     }
3427
3428   sm->max_translations_per_fib[fib_index] = session_limit;
3429   return 0;
3430 }
3431
3432 int
3433 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3434 {
3435   snat_main_t *sm = &snat_main;
3436
3437   if (nat44_set_session_limit (session_limit, vrf_id))
3438     return 1;
3439   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3440
3441   sm->translation_buckets =
3442     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3443
3444   nat44_sessions_clear ();
3445   return 0;
3446 }
3447
3448 static void
3449 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
3450                          u32 translation_buckets, u32 user_buckets)
3451 {
3452   dlist_elt_t *head;
3453
3454   pool_alloc (tsm->sessions, translations);
3455   pool_alloc (tsm->lru_pool, translations);
3456
3457   pool_get (tsm->lru_pool, head);
3458   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3459   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3460
3461   pool_get (tsm->lru_pool, head);
3462   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3463   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3464
3465   pool_get (tsm->lru_pool, head);
3466   tsm->udp_lru_head_index = head - tsm->lru_pool;
3467   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3468
3469   pool_get (tsm->lru_pool, head);
3470   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3471   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3472
3473   pool_get (tsm->lru_pool, head);
3474   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3475   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3476
3477   // TODO: ED nat is not using these
3478   // before removal large refactor required
3479   pool_alloc (tsm->list_pool, translations);
3480   clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets, 0);
3481   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3482 }
3483
3484 static void
3485 reinit_ed_flow_hash ()
3486 {
3487   snat_main_t *sm = &snat_main;
3488   // we expect 2 flows per session, so multiply translation_buckets by 2
3489   clib_bihash_init_16_8 (
3490     &sm->flow_hash, "ed-flow-hash",
3491     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3492   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3493 }
3494
3495 static void
3496 nat44_ed_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
3497 {
3498   snat_main_t *sm = &snat_main;
3499   snat_main_per_thread_data_t *tsm;
3500   u32 static_mapping_buckets = 1024;
3501   u32 static_mapping_memory_size = 64 << 20;
3502
3503   reinit_ed_flow_hash ();
3504
3505   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3506                         "static_mapping_by_local", static_mapping_buckets,
3507                         static_mapping_memory_size);
3508   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3509                                      format_static_mapping_kvp);
3510
3511   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3512                         "static_mapping_by_external", static_mapping_buckets,
3513                         static_mapping_memory_size);
3514   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3515                                      format_static_mapping_kvp);
3516
3517   if (sm->pat)
3518     {
3519       vec_foreach (tsm, sm->per_thread_data)
3520         {
3521           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3522                                    sm->translation_buckets, sm->user_buckets);
3523         }
3524     }
3525 }
3526
3527 static void
3528 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3529 {
3530   pool_free (tsm->sessions);
3531   pool_free (tsm->lru_pool);
3532
3533   vec_free (tsm->per_vrf_sessions_vec);
3534
3535   // TODO: resolve static mappings (put only to !ED)
3536   pool_free (tsm->list_pool);
3537   pool_free (tsm->users);
3538   clib_bihash_free_8_8 (&tsm->user_hash);
3539 }
3540
3541 static void
3542 nat44_ed_db_free ()
3543 {
3544   snat_main_t *sm = &snat_main;
3545   snat_main_per_thread_data_t *tsm;
3546
3547   pool_free (sm->static_mappings);
3548   clib_bihash_free_16_8 (&sm->flow_hash);
3549   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
3550   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
3551
3552   if (sm->pat)
3553     {
3554       vec_foreach (tsm, sm->per_thread_data)
3555         {
3556           nat44_ed_worker_db_free (tsm);
3557         }
3558     }
3559 }
3560
3561 void
3562 nat44_ed_sessions_clear ()
3563 {
3564   snat_main_t *sm = &snat_main;
3565   snat_main_per_thread_data_t *tsm;
3566
3567   reinit_ed_flow_hash ();
3568
3569   if (sm->pat)
3570     {
3571       vec_foreach (tsm, sm->per_thread_data)
3572         {
3573
3574           nat44_ed_worker_db_free (tsm);
3575           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3576                                    sm->translation_buckets, sm->user_buckets);
3577         }
3578     }
3579
3580   // TODO: function for reset counters
3581   vlib_zero_simple_counter (&sm->total_users, 0);
3582   vlib_zero_simple_counter (&sm->total_sessions, 0);
3583   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
3584 }
3585
3586 void
3587 nat44_sessions_clear ()
3588 {
3589   snat_main_t *sm = &snat_main;
3590
3591   if (sm->endpoint_dependent)
3592     nat44_ed_sessions_clear ();
3593   else
3594     nat44_ei_sessions_clear ();
3595 }
3596
3597 static void
3598 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3599                                  uword opaque,
3600                                  u32 sw_if_index,
3601                                  ip4_address_t * address,
3602                                  u32 address_length,
3603                                  u32 if_address_index, u32 is_delete)
3604 {
3605   snat_main_t *sm = &snat_main;
3606   snat_static_map_resolve_t *rp;
3607   snat_static_mapping_t *m;
3608   clib_bihash_kv_8_8_t kv, value;
3609   int i, rv;
3610   ip4_address_t l_addr;
3611
3612   if (!sm->enabled)
3613     return;
3614
3615   for (i = 0; i < vec_len (sm->to_resolve); i++)
3616     {
3617       rp = sm->to_resolve + i;
3618       if (rp->addr_only == 0)
3619         continue;
3620       if (rp->sw_if_index == sw_if_index)
3621         goto match;
3622     }
3623
3624   return;
3625
3626 match:
3627   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
3628               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
3629   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3630     m = 0;
3631   else
3632     m = pool_elt_at_index (sm->static_mappings, value.value);
3633
3634   if (!is_delete)
3635     {
3636       /* Don't trip over lease renewal, static config */
3637       if (m)
3638         return;
3639     }
3640   else
3641     {
3642       if (!m)
3643         return;
3644     }
3645
3646   /* Indetity mapping? */
3647   if (rp->l_addr.as_u32 == 0)
3648     l_addr.as_u32 = address[0].as_u32;
3649   else
3650     l_addr.as_u32 = rp->l_addr.as_u32;
3651   /* Add the static mapping */
3652   rv = snat_add_static_mapping (l_addr,
3653                                 address[0],
3654                                 rp->l_port,
3655                                 rp->e_port,
3656                                 rp->vrf_id,
3657                                 rp->addr_only, ~0 /* sw_if_index */ ,
3658                                 rp->proto, !is_delete, rp->twice_nat,
3659                                 rp->out2in_only, rp->tag, rp->identity_nat,
3660                                 rp->pool_addr, rp->exact);
3661   if (rv)
3662     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
3663 }
3664
3665 static void
3666 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3667                                        uword opaque,
3668                                        u32 sw_if_index,
3669                                        ip4_address_t * address,
3670                                        u32 address_length,
3671                                        u32 if_address_index, u32 is_delete)
3672 {
3673   snat_main_t *sm = &snat_main;
3674   snat_static_map_resolve_t *rp;
3675   ip4_address_t l_addr;
3676   int i, j;
3677   int rv;
3678   u8 twice_nat = 0;
3679   snat_address_t *addresses = sm->addresses;
3680
3681   if (!sm->enabled)
3682     return;
3683
3684   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3685     {
3686       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3687         goto match;
3688     }
3689
3690   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3691     {
3692       twice_nat = 1;
3693       addresses = sm->twice_nat_addresses;
3694       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3695         goto match;
3696     }
3697
3698   return;
3699
3700 match:
3701   if (!is_delete)
3702     {
3703       /* Don't trip over lease renewal, static config */
3704       for (j = 0; j < vec_len (addresses); j++)
3705         if (addresses[j].addr.as_u32 == address->as_u32)
3706           return;
3707
3708       (void) snat_add_address (sm, address, ~0, twice_nat);
3709       /* Scan static map resolution vector */
3710       for (j = 0; j < vec_len (sm->to_resolve); j++)
3711         {
3712           rp = sm->to_resolve + j;
3713           if (rp->addr_only)
3714             continue;
3715           /* On this interface? */
3716           if (rp->sw_if_index == sw_if_index)
3717             {
3718               /* Indetity mapping? */
3719               if (rp->l_addr.as_u32 == 0)
3720                 l_addr.as_u32 = address[0].as_u32;
3721               else
3722                 l_addr.as_u32 = rp->l_addr.as_u32;
3723               /* Add the static mapping */
3724               rv = snat_add_static_mapping (
3725                 l_addr, address[0], rp->l_port, rp->e_port, rp->vrf_id,
3726                 rp->addr_only, ~0 /* sw_if_index */, rp->proto, 1,
3727                 rp->twice_nat, rp->out2in_only, rp->tag, rp->identity_nat,
3728                 rp->pool_addr, rp->exact);
3729               if (rv)
3730                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
3731                                     "i4", rv);
3732             }
3733         }
3734       return;
3735     }
3736   else
3737     {
3738       (void) snat_del_address (sm, address[0], 1, twice_nat);
3739       return;
3740     }
3741 }
3742
3743 int
3744 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3745                             u8 twice_nat)
3746 {
3747   ip4_main_t *ip4_main = sm->ip4_main;
3748   ip4_address_t *first_int_addr;
3749   snat_static_map_resolve_t *rp;
3750   u32 *indices_to_delete = 0;
3751   int i, j;
3752   u32 *auto_add_sw_if_indices =
3753     twice_nat ? sm->
3754     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3755
3756   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3757     );
3758
3759   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3760     {
3761       if (auto_add_sw_if_indices[i] == sw_if_index)
3762         {
3763           if (is_del)
3764             {
3765               /* if have address remove it */
3766               if (first_int_addr)
3767                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3768               else
3769                 {
3770                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3771                     {
3772                       rp = sm->to_resolve + j;
3773                       if (rp->sw_if_index == sw_if_index)
3774                         vec_add1 (indices_to_delete, j);
3775                     }
3776                   if (vec_len (indices_to_delete))
3777                     {
3778                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3779                         vec_del1 (sm->to_resolve, j);
3780                       vec_free (indices_to_delete);
3781                     }
3782                 }
3783               if (twice_nat)
3784                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3785               else
3786                 vec_del1 (sm->auto_add_sw_if_indices, i);
3787             }
3788           else
3789             return VNET_API_ERROR_VALUE_EXIST;
3790
3791           return 0;
3792         }
3793     }
3794
3795   if (is_del)
3796     return VNET_API_ERROR_NO_SUCH_ENTRY;
3797
3798   /* add to the auto-address list */
3799   if (twice_nat)
3800     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3801   else
3802     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3803
3804   /* If the address is already bound - or static - add it now */
3805   if (first_int_addr)
3806     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3807
3808   return 0;
3809 }
3810
3811 int
3812 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3813                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3814                       u32 vrf_id, int is_in)
3815 {
3816   ip4_header_t ip;
3817   clib_bihash_kv_16_8_t kv, value;
3818   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3819   snat_session_t *s;
3820   snat_main_per_thread_data_t *tsm;
3821
3822   if (!sm->endpoint_dependent)
3823     return VNET_API_ERROR_FEATURE_DISABLED;
3824
3825   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3826   if (sm->num_workers > 1)
3827     tsm =
3828       vec_elt_at_index (sm->per_thread_data,
3829                         sm->worker_in2out_cb (&ip, fib_index, 0));
3830   else
3831     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3832
3833   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
3834   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3835     {
3836       return VNET_API_ERROR_NO_SUCH_ENTRY;
3837     }
3838
3839   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3840     return VNET_API_ERROR_UNSPECIFIED;
3841   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3842   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3843   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3844   return 0;
3845 }
3846
3847 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3848                                  vlib_node_runtime_t * node,
3849                                  vlib_frame_t * frame)
3850 {
3851   return 0;
3852 }
3853
3854 /* *INDENT-OFF* */
3855 VLIB_REGISTER_NODE (nat_default_node) = {
3856   .name = "nat-default",
3857   .vector_size = sizeof (u32),
3858   .format_trace = 0,
3859   .type = VLIB_NODE_TYPE_INTERNAL,
3860   .n_errors = 0,
3861   .n_next_nodes = NAT_N_NEXT,
3862   .next_nodes = {
3863     [NAT_NEXT_DROP] = "error-drop",
3864     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3865     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3866     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3867     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3868     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3869     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3870     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3871     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3872     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3873   },
3874 };
3875 /* *INDENT-ON* */
3876
3877 void
3878 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3879 {
3880   f->l3_csum_delta = 0;
3881   f->l4_csum_delta = 0;
3882   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3883       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3884     {
3885       f->l3_csum_delta =
3886         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3887       f->l3_csum_delta =
3888         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3889     }
3890   else
3891     {
3892       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3893     }
3894   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3895       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3896     {
3897       f->l3_csum_delta =
3898         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3899       f->l3_csum_delta =
3900         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3901     }
3902   else
3903     {
3904       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3905     }
3906   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3907     {
3908       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3909       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3910     }
3911   else
3912     {
3913       f->rewrite.sport = f->match.sport;
3914     }
3915   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3916     {
3917       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3918       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3919     }
3920   else
3921     {
3922       f->rewrite.dport = f->match.dport;
3923     }
3924   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3925       f->rewrite.icmp_id != f->match.icmp_id)
3926     {
3927       f->l4_csum_delta =
3928         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3929       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.icmp_id);
3930     }
3931   else
3932     {
3933       f->rewrite.icmp_id = f->match.icmp_id;
3934     }
3935   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3936     {
3937     }
3938   else
3939     {
3940       f->rewrite.fib_index = f->match.fib_index;
3941     }
3942 }
3943
3944 static_always_inline int nat_6t_flow_icmp_translate (snat_main_t *sm,
3945                                                      vlib_buffer_t *b,
3946                                                      ip4_header_t *ip,
3947                                                      nat_6t_flow_t *f);
3948
3949 static_always_inline void
3950 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3951                            nat_6t_flow_t *f, nat_protocol_t proto,
3952                            int is_icmp_inner_ip4)
3953 {
3954   udp_header_t *udp = ip4_next_header (ip);
3955   tcp_header_t *tcp = (tcp_header_t *) udp;
3956
3957   if ((NAT_PROTOCOL_TCP == proto || NAT_PROTOCOL_UDP == proto) &&
3958       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3959     {
3960       if (!is_icmp_inner_ip4)
3961         { // regular case
3962           ip->src_address = f->rewrite.saddr;
3963           ip->dst_address = f->rewrite.daddr;
3964           udp->src_port = f->rewrite.sport;
3965           udp->dst_port = f->rewrite.dport;
3966         }
3967       else
3968         { // icmp inner ip4 - reversed saddr/daddr
3969           ip->src_address = f->rewrite.daddr;
3970           ip->dst_address = f->rewrite.saddr;
3971           udp->src_port = f->rewrite.dport;
3972           udp->dst_port = f->rewrite.sport;
3973         }
3974
3975       if (NAT_PROTOCOL_TCP == proto)
3976         {
3977           ip_csum_t tcp_sum = tcp->checksum;
3978           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3979           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3980           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3981           tcp->checksum = ip_csum_fold (tcp_sum);
3982         }
3983       else if (proto == NAT_PROTOCOL_UDP && udp->checksum)
3984         {
3985           ip_csum_t udp_sum = udp->checksum;
3986           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3987           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3988           udp->checksum = ip_csum_fold (udp_sum);
3989         }
3990     }
3991   else
3992     {
3993       if (!is_icmp_inner_ip4)
3994         { // regular case
3995           ip->src_address = f->rewrite.saddr;
3996           ip->dst_address = f->rewrite.daddr;
3997         }
3998       else
3999         { // icmp inner ip4 - reversed saddr/daddr
4000           ip->src_address = f->rewrite.daddr;
4001           ip->dst_address = f->rewrite.saddr;
4002         }
4003     }
4004
4005   ip_csum_t ip_sum = ip->checksum;
4006   ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
4007   ip->checksum = ip_csum_fold (ip_sum);
4008   ASSERT (ip->checksum == ip4_header_checksum (ip));
4009 }
4010
4011 static_always_inline int
4012 nat_6t_flow_icmp_translate (snat_main_t *sm, vlib_buffer_t *b,
4013                             ip4_header_t *ip, nat_6t_flow_t *f)
4014 {
4015   if (IP_PROTOCOL_ICMP != ip->protocol)
4016     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4017
4018   icmp46_header_t *icmp = ip4_next_header (ip);
4019   icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
4020
4021   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
4022     {
4023       if (icmp->checksum == 0)
4024         icmp->checksum = 0xffff;
4025
4026       if (!icmp_type_is_error_message (icmp->type))
4027         {
4028           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
4029               (f->rewrite.icmp_id != echo->identifier))
4030             {
4031               ip_csum_t sum = icmp->checksum;
4032               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
4033                                     icmp_echo_header_t,
4034                                     identifier /* changed member */);
4035               echo->identifier = f->rewrite.icmp_id;
4036               icmp->checksum = ip_csum_fold (sum);
4037             }
4038         }
4039       else
4040         {
4041           // errors are not fragmented
4042           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
4043
4044           if (!ip4_header_checksum_is_valid (inner_ip))
4045             {
4046               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4047             }
4048
4049           nat_protocol_t inner_proto =
4050             ip_proto_to_nat_proto (inner_ip->protocol);
4051
4052           ip_csum_t icmp_sum = icmp->checksum;
4053
4054           switch (inner_proto)
4055             {
4056             case NAT_PROTOCOL_UDP:
4057             case NAT_PROTOCOL_TCP:
4058               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
4059                                          1 /* is_icmp_inner_ip4 */);
4060               icmp_sum = ip_csum_sub_even (icmp_sum, f->l3_csum_delta);
4061               icmp->checksum = ip_csum_fold (icmp_sum);
4062               break;
4063             case NAT_PROTOCOL_ICMP:
4064               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
4065                 {
4066                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
4067                   icmp_echo_header_t *inner_echo =
4068                     (icmp_echo_header_t *) (inner_icmp + 1);
4069                   if (f->rewrite.icmp_id != inner_echo->identifier)
4070                     {
4071                       ip_csum_t sum = icmp->checksum;
4072                       sum = ip_csum_update (
4073                         sum, inner_echo->identifier, f->rewrite.icmp_id,
4074                         icmp_echo_header_t, identifier /* changed member */);
4075                       icmp->checksum = ip_csum_fold (sum);
4076                       ip_csum_t inner_sum = inner_icmp->checksum;
4077                       inner_sum = ip_csum_update (
4078                         sum, inner_echo->identifier, f->rewrite.icmp_id,
4079                         icmp_echo_header_t, identifier /* changed member */);
4080                       inner_icmp->checksum = ip_csum_fold (inner_sum);
4081                       inner_echo->identifier = f->rewrite.icmp_id;
4082                     }
4083                 }
4084               break;
4085             default:
4086               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
4087               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4088             }
4089         }
4090     }
4091   return NAT_ED_TRNSL_ERR_SUCCESS;
4092 }
4093
4094 nat_translation_error_e
4095 nat_6t_flow_buf_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
4096                            nat_6t_flow_t *f, nat_protocol_t proto,
4097                            int is_output_feature)
4098 {
4099   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
4100     {
4101       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
4102     }
4103
4104   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */);
4105
4106   if (NAT_PROTOCOL_ICMP == proto)
4107     {
4108       return nat_6t_flow_icmp_translate (sm, b, ip, f);
4109     }
4110
4111   return NAT_ED_TRNSL_ERR_SUCCESS;
4112 }
4113
4114 u8 *
4115 format_nat_6t (u8 *s, va_list *args)
4116 {
4117   nat_6t_t *t = va_arg (*args, nat_6t_t *);
4118
4119   s = format (s, "saddr %U sport %u daddr %U dport %u proto %U fib_idx %u",
4120               format_ip4_address, t->saddr.as_u8,
4121               clib_net_to_host_u16 (t->sport), format_ip4_address,
4122               t->daddr.as_u8, clib_net_to_host_u16 (t->dport),
4123               format_ip_protocol, t->proto, t->fib_index);
4124   return s;
4125 }
4126
4127 u8 *
4128 format_nat_ed_translation_error (u8 *s, va_list *args)
4129 {
4130   nat_translation_error_e e = va_arg (*args, nat_translation_error_e);
4131
4132   switch (e)
4133     {
4134     case NAT_ED_TRNSL_ERR_SUCCESS:
4135       s = format (s, "success");
4136       break;
4137     case NAT_ED_TRNSL_ERR_TRANSLATION_FAILED:
4138       s = format (s, "translation-failed");
4139       break;
4140     case NAT_ED_TRNSL_ERR_FLOW_MISMATCH:
4141       s = format (s, "flow-mismatch");
4142       break;
4143     }
4144   return s;
4145 }
4146
4147 u8 *
4148 format_nat_6t_flow (u8 *s, va_list *args)
4149 {
4150   nat_6t_flow_t *f = va_arg (*args, nat_6t_flow_t *);
4151
4152   s = format (s, "match: %U ", format_nat_6t, &f->match);
4153   int r = 0;
4154   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE)
4155     {
4156       s = format (s, "rewrite: saddr %U ", format_ip4_address,
4157                   f->rewrite.saddr.as_u8);
4158       r = 1;
4159     }
4160   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE)
4161     {
4162       if (!r)
4163         {
4164           s = format (s, "rewrite: ");
4165           r = 1;
4166         }
4167       s = format (s, "sport %u ", clib_net_to_host_u16 (f->rewrite.sport));
4168     }
4169   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE)
4170     {
4171       if (!r)
4172         {
4173           s = format (s, "rewrite: ");
4174           r = 1;
4175         }
4176       s = format (s, "daddr %U ", format_ip4_address, f->rewrite.daddr.as_u8);
4177     }
4178   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE)
4179     {
4180       if (!r)
4181         {
4182           s = format (s, "rewrite: ");
4183           r = 1;
4184         }
4185       s = format (s, "dport %u ", clib_net_to_host_u16 (f->rewrite.dport));
4186     }
4187   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
4188     {
4189       if (!r)
4190         {
4191           s = format (s, "rewrite: ");
4192           r = 1;
4193         }
4194       s = format (s, "icmp-id %u ", clib_net_to_host_u16 (f->rewrite.icmp_id));
4195     }
4196   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
4197     {
4198       if (!r)
4199         {
4200           s = format (s, "rewrite: ");
4201           r = 1;
4202         }
4203       s = format (s, "txfib %u ", f->rewrite.fib_index);
4204     }
4205   return s;
4206 }
4207
4208 /*
4209  * fd.io coding-style-patch-verification: ON
4210  *
4211  * Local Variables:
4212  * eval: (c-set-style "gnu")
4213  * End:
4214  */