src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/lib/ipfix_logging.h>
  25 #include <nat/lib/nat_syslog.h>
  26 #include <nat/nat_inlines.h>
  27 #include <nat/nat44/inlines.h>
  28 #include <nat/nat_affinity.h>
  29 #include <vnet/fib/fib_table.h>
  30 #include <vnet/fib/ip4_fib.h>
  31 #include <vnet/ip/reass/ip4_sv_reass.h>
  32 #include <vppinfra/bihash_16_8.h>
  33 #include <nat/nat44/ed_inlines.h>
  34 #include <vnet/ip/ip_table.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_ha.h>
  37 #include <nat/nat44-ei/nat44_ei.h>
  38
  39 #include <vpp/app/version.h>
  40 #include <nat/lib/nat_inlines.h>
  41
  42 snat_main_t snat_main;
  43
  44 #define skip_if_disabled()                                                    \
  45   do                                                                          \
  46     {                                                                         \
  47       snat_main_t *sm = &snat_main;                                           \
  48       if (PREDICT_FALSE (!sm->enabled))                                       \
  49         return;                                                               \
  50     }                                                                         \
  51   while (0)
  52
  53 #define fail_if_enabled()                                                     \
  54   do                                                                          \
  55     {                                                                         \
  56       snat_main_t *sm = &snat_main;                                           \
  57       if (PREDICT_FALSE (sm->enabled))                                        \
  58         {                                                                     \
  59           nat_log_err ("plugin enabled");                                     \
  60           return 1;                                                           \
  61         }                                                                     \
  62     }                                                                         \
  63   while (0)
  64
  65 #define fail_if_disabled()                                                    \
  66   do                                                                          \
  67     {                                                                         \
  68       snat_main_t *sm = &snat_main;                                           \
  69       if (PREDICT_FALSE (!sm->enabled))                                       \
  70         {                                                                     \
  71           nat_log_err ("plugin disabled");                                    \
  72           return 1;                                                           \
  73         }                                                                     \
  74     }                                                                         \
  75   while (0)
  76
  77 /* *INDENT-OFF* */
  78 /* Hook up input features */
  79 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat-pre-in2out",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  83                                "ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat-pre-out2in",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  89                                "ip4-dhcp-client-detect",
  90                                "ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-in2out-worker-handoff",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  96 };
  97 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-out2in-worker-handoff",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 101                                "ip4-dhcp-client-detect"),
 102 };
 103 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-in2out",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-out2in",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 112                                "ip4-dhcp-client-detect"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-classify",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 118 };
 119 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-ed-in2out",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 123 };
 124 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-ed-out2in",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 128                                "ip4-dhcp-client-detect"),
 129 };
 130 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 131   .arc_name = "ip4-unicast",
 132   .node_name = "nat44-ed-classify",
 133   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 134 };
 135 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 136   .arc_name = "ip4-unicast",
 137   .node_name = "nat44-handoff-classify",
 138   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 141   .arc_name = "ip4-unicast",
 142   .node_name = "nat44-in2out-fast",
 143   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 144 };
 145 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 146   .arc_name = "ip4-unicast",
 147   .node_name = "nat44-out2in-fast",
 148   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 149                                "ip4-dhcp-client-detect"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 152   .arc_name = "ip4-unicast",
 153   .node_name = "nat44-hairpin-dst",
 154   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 155 };
 156
 157 /* Hook up output features */
 158 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 159   .arc_name = "ip4-output",
 160   .node_name = "nat44-in2out-output",
 161   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 162 };
 163 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 164   .arc_name = "ip4-output",
 165   .node_name = "nat44-in2out-output-worker-handoff",
 166   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 167 };
 168 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 169   .arc_name = "ip4-output",
 170   .node_name = "nat44-hairpin-src",
 171   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 172 };
 173 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 174   .arc_name = "ip4-output",
 175   .node_name = "nat-pre-in2out-output",
 176   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 177   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 178 };
 179 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 180   .arc_name = "ip4-output",
 181   .node_name = "nat44-ed-in2out-output",
 182   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 183   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 184 };
 185
 186 /* Hook up ip4-local features */
 187 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 188 {
 189   .arc_name = "ip4-local",
 190   .node_name = "nat44-hairpinning",
 191   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 192 };
 193
 194 VLIB_PLUGIN_REGISTER () = {
 195     .version = VPP_BUILD_VER,
 196     .description = "Network Address Translation (NAT)",
 197 };
 198 /* *INDENT-ON* */
 199
 200 static void nat44_ed_db_init (u32 translations, u32 translation_buckets,
 201                               u32 user_buckets);
 202
 203 static void nat44_ed_db_free ();
 204
 205 static u32
 206 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
 207                                u32 rx_fib_index, u8 is_output);
 208
 209 static u32
 210 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
 211                                u8 is_output);
 212
 213 u32 nat_calc_bihash_buckets (u32 n_elts);
 214
 215 u8 *
 216 format_session_kvp (u8 * s, va_list * args)
 217 {
 218   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 219
 220   s = format (s, "%U thread-index %llu session-index %llu", format_snat_key,
 221               v->key, nat_value_get_thread_index (v),
 222               nat_value_get_session_index (v));
 223
 224   return s;
 225 }
 226
 227 u8 *
 228 format_static_mapping_kvp (u8 * s, va_list * args)
 229 {
 230   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 231
 232   s = format (s, "%U static-mapping-index %llu",
 233               format_snat_key, v->key, v->value);
 234
 235   return s;
 236 }
 237
 238 u8 *
 239 format_user_kvp (u8 * s, va_list * args)
 240 {
 241   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 242   snat_user_key_t k;
 243
 244   k.as_u64 = v->key;
 245
 246   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
 247               k.fib_index, v->value);
 248
 249   return s;
 250 }
 251
 252 u8 *
 253 format_ed_session_kvp (u8 * s, va_list * args)
 254 {
 255   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 256
 257   u8 proto;
 258   u16 r_port, l_port;
 259   ip4_address_t l_addr, r_addr;
 260   u32 fib_index;
 261
 262   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 263   s = format (s,
 264               "local %U:%d remote %U:%d proto %U fib %d thread-index %u "
 265               "session-index %u",
 266               format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 267               format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 268               format_ip_protocol, proto, fib_index,
 269               ed_value_get_thread_index (v), ed_value_get_session_index (v));
 270
 271   return s;
 272 }
 273
 274 void
 275 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 276                        u8 is_ha)
 277 {
 278   clib_bihash_kv_8_8_t kv;
 279
 280   if (is_ed_session (s))
 281     {
 282       per_vrf_sessions_unregister_session (s, thread_index);
 283
 284       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 285         nat_elog_warn ("flow hash del failed");
 286
 287       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 288         nat_elog_warn ("flow hash del failed");
 289     }
 290
 291   if (is_fwd_bypass_session (s))
 292     {
 293       return;
 294     }
 295
 296   /* session lookup tables */
 297   if (is_ed_session (s))
 298     {
 299       if (is_affinity_sessions (s))
 300         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 301                              s->nat_proto, s->out2in.port);
 302
 303       if (!is_ha)
 304         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 305                                &s->in2out.addr, s->in2out.port,
 306                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 307                                &s->out2in.addr, s->out2in.port,
 308                                &s->ext_host_addr, s->ext_host_port,
 309                                s->nat_proto, is_twice_nat_session (s));
 310     }
 311   else
 312     {
 313       init_nat_i2o_k (&kv, s);
 314       if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
 315         nat_elog_warn ("in2out key del failed");
 316       init_nat_o2i_k (&kv, s);
 317       if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
 318         nat_elog_warn ("out2in key del failed");
 319
 320       if (!is_ha)
 321         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 322                                  &s->in2out.addr, s->in2out.port,
 323                                  &s->out2in.addr, s->out2in.port,
 324                                  s->nat_proto);
 325     }
 326
 327   if (snat_is_unk_proto_session (s))
 328     return;
 329
 330   if (!is_ha)
 331     {
 332       /* log NAT event */
 333       nat_ipfix_logging_nat44_ses_delete (thread_index,
 334                                           s->in2out.addr.as_u32,
 335                                           s->out2in.addr.as_u32,
 336                                           s->nat_proto,
 337                                           s->in2out.port,
 338                                           s->out2in.port,
 339                                           s->in2out.fib_index);
 340
 341       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 342                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 343                    thread_index);
 344     }
 345
 346   /* Twice NAT address and port for external host */
 347   if (is_twice_nat_session (s))
 348     {
 349       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 350                                           thread_index,
 351                                           &s->ext_host_nat_addr,
 352                                           s->ext_host_nat_port, s->nat_proto);
 353     }
 354
 355   if (snat_is_session_static (s))
 356     return;
 357
 358   snat_free_outside_address_and_port (sm->addresses, thread_index,
 359                                       &s->out2in.addr, s->out2in.port,
 360                                       s->nat_proto);
 361 }
 362
 363 snat_user_t *
 364 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 365                         u32 thread_index)
 366 {
 367   snat_user_t *u = 0;
 368   snat_user_key_t user_key;
 369   clib_bihash_kv_8_8_t kv, value;
 370   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 371   dlist_elt_t *per_user_list_head_elt;
 372
 373   user_key.addr.as_u32 = addr->as_u32;
 374   user_key.fib_index = fib_index;
 375   kv.key = user_key.as_u64;
 376
 377   /* Ever heard of the "user" = src ip4 address before? */
 378   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 379     {
 380       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 381         {
 382           vlib_increment_simple_counter (&sm->user_limit_reached,
 383                                          thread_index, 0, 1);
 384           nat_elog_warn ("maximum user limit reached");
 385           return NULL;
 386         }
 387       /* no, make a new one */
 388       pool_get (tsm->users, u);
 389       clib_memset (u, 0, sizeof (*u));
 390
 391       u->addr.as_u32 = addr->as_u32;
 392       u->fib_index = fib_index;
 393
 394       pool_get (tsm->list_pool, per_user_list_head_elt);
 395
 396       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 397         tsm->list_pool;
 398
 399       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 400
 401       kv.value = u - tsm->users;
 402
 403       /* add user */
 404       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 405         {
 406           nat_elog_warn ("user_hash key add failed");
 407           nat44_delete_user_with_no_session (sm, u, thread_index);
 408           return NULL;
 409         }
 410
 411       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 412                                pool_elts (tsm->users));
 413     }
 414   else
 415     {
 416       u = pool_elt_at_index (tsm->users, value.value);
 417     }
 418
 419   return u;
 420 }
 421
 422 // only NAT EI
 423 snat_session_t *
 424 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 425                               u32 thread_index, f64 now)
 426 {
 427   snat_session_t *s;
 428   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 429   u32 oldest_per_user_translation_list_index, session_index;
 430   dlist_elt_t *oldest_per_user_translation_list_elt;
 431   dlist_elt_t *per_user_translation_list_elt;
 432
 433   /* Over quota? Recycle the least recently used translation */
 434   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 435     {
 436       oldest_per_user_translation_list_index =
 437         clib_dlist_remove_head (tsm->list_pool,
 438                                 u->sessions_per_user_list_head_index);
 439
 440       ASSERT (oldest_per_user_translation_list_index != ~0);
 441
 442       /* Add it back to the end of the LRU list */
 443       clib_dlist_addtail (tsm->list_pool,
 444                           u->sessions_per_user_list_head_index,
 445                           oldest_per_user_translation_list_index);
 446       /* Get the list element */
 447       oldest_per_user_translation_list_elt =
 448         pool_elt_at_index (tsm->list_pool,
 449                            oldest_per_user_translation_list_index);
 450
 451       /* Get the session index from the list element */
 452       session_index = oldest_per_user_translation_list_elt->value;
 453
 454       /* Get the session */
 455       s = pool_elt_at_index (tsm->sessions, session_index);
 456
 457       // TODO: ONLY EI version should be called
 458       nat_free_session_data (sm, s, thread_index, 0);
 459       if (snat_is_session_static (s))
 460         u->nstaticsessions--;
 461       else
 462         u->nsessions--;
 463       s->flags = 0;
 464       s->total_bytes = 0;
 465       s->total_pkts = 0;
 466       s->state = 0;
 467       s->ext_host_addr.as_u32 = 0;
 468       s->ext_host_port = 0;
 469       s->ext_host_nat_addr.as_u32 = 0;
 470       s->ext_host_nat_port = 0;
 471     }
 472   else
 473     {
 474       pool_get (tsm->sessions, s);
 475       clib_memset (s, 0, sizeof (*s));
 476
 477       /* Create list elts */
 478       pool_get (tsm->list_pool, per_user_translation_list_elt);
 479       clib_dlist_init (tsm->list_pool,
 480                        per_user_translation_list_elt - tsm->list_pool);
 481
 482       per_user_translation_list_elt->value = s - tsm->sessions;
 483       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 484       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 485
 486       clib_dlist_addtail (tsm->list_pool,
 487                           s->per_user_list_head_index,
 488                           per_user_translation_list_elt - tsm->list_pool);
 489
 490       s->user_index = u - tsm->users;
 491       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 492                                pool_elts (tsm->sessions));
 493     }
 494
 495   s->ha_last_refreshed = now;
 496
 497   return s;
 498 }
 499
 500 void
 501 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 502                           int is_add)
 503 {
 504   snat_main_t *sm = &snat_main;
 505   fib_prefix_t prefix = {
 506     .fp_len = p_len,
 507     .fp_proto = FIB_PROTOCOL_IP4,
 508     .fp_addr = {
 509                 .ip4.as_u32 = addr->as_u32,
 510                 },
 511   };
 512   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 513
 514   if (is_add)
 515     fib_table_entry_update_one_path (fib_index,
 516                                      &prefix,
 517                                      sm->fib_src_low,
 518                                      (FIB_ENTRY_FLAG_CONNECTED |
 519                                       FIB_ENTRY_FLAG_LOCAL |
 520                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 521                                      DPO_PROTO_IP4,
 522                                      NULL,
 523                                      sw_if_index,
 524                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 525   else
 526     fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 527 }
 528
 529 int
 530 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 531                   u8 twice_nat)
 532 {
 533   snat_address_t *ap;
 534   snat_interface_t *i;
 535   vlib_thread_main_t *tm = vlib_get_thread_main ();
 536
 537   if (twice_nat && !sm->endpoint_dependent)
 538     {
 539       nat_log_err ("unsupported");
 540       return VNET_API_ERROR_UNSUPPORTED;
 541     }
 542
 543   /* Check if address already exists */
 544   /* *INDENT-OFF* */
 545   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 546     {
 547       if (ap->addr.as_u32 == addr->as_u32)
 548         {
 549           nat_log_err ("address exist");
 550           return VNET_API_ERROR_VALUE_EXIST;
 551         }
 552     }
 553   /* *INDENT-ON* */
 554
 555   if (twice_nat)
 556     vec_add2 (sm->twice_nat_addresses, ap, 1);
 557   else
 558     vec_add2 (sm->addresses, ap, 1);
 559
 560   ap->addr = *addr;
 561   if (vrf_id != ~0)
 562     ap->fib_index =
 563       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 564                                          sm->fib_src_low);
 565   else
 566     ap->fib_index = ~0;
 567
 568   /* *INDENT-OFF* */
 569   #define _(N, i, n, s) \
 570     clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 571     ap->busy_##n##_ports = 0; \
 572     ap->busy_##n##_ports_per_thread = 0;\
 573     vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 574     foreach_nat_protocol
 575   #undef _
 576   /* *INDENT-ON* */
 577
 578   if (twice_nat)
 579     return 0;
 580
 581   /* Add external address to FIB */
 582   /* *INDENT-OFF* */
 583   pool_foreach (i, sm->interfaces)
 584    {
 585     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 586       continue;
 587
 588     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 589     break;
 590   }
 591   pool_foreach (i, sm->output_feature_interfaces)
 592    {
 593     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 594       continue;
 595
 596     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 597     break;
 598   }
 599   /* *INDENT-ON* */
 600
 601   return 0;
 602 }
 603
 604 static int
 605 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 606 {
 607   snat_static_mapping_t *m;
 608   /* *INDENT-OFF* */
 609   pool_foreach (m, sm->static_mappings)
 610    {
 611       if (is_addr_only_static_mapping (m) ||
 612           is_out2in_only_static_mapping (m) ||
 613           is_identity_static_mapping (m))
 614         continue;
 615       if (m->external_addr.as_u32 == addr.as_u32)
 616         return 1;
 617   }
 618   /* *INDENT-ON* */
 619
 620   return 0;
 621 }
 622
 623 static void
 624 snat_add_static_mapping_when_resolved (snat_main_t *sm, ip4_address_t l_addr,
 625                                        u16 l_port, u32 sw_if_index, u16 e_port,
 626                                        u32 vrf_id, nat_protocol_t proto,
 627                                        int addr_only, u8 *tag, int twice_nat,
 628                                        int out2in_only, int identity_nat,
 629                                        ip4_address_t pool_addr, int exact)
 630 {
 631   snat_static_map_resolve_t *rp;
 632
 633   vec_add2 (sm->to_resolve, rp, 1);
 634   rp->l_addr.as_u32 = l_addr.as_u32;
 635   rp->l_port = l_port;
 636   rp->sw_if_index = sw_if_index;
 637   rp->e_port = e_port;
 638   rp->vrf_id = vrf_id;
 639   rp->proto = proto;
 640   rp->addr_only = addr_only;
 641   rp->twice_nat = twice_nat;
 642   rp->out2in_only = out2in_only;
 643   rp->identity_nat = identity_nat;
 644   rp->tag = vec_dup (tag);
 645   rp->pool_addr = pool_addr;
 646   rp->exact = exact;
 647 }
 648
 649 u32
 650 get_thread_idx_by_port (u16 e_port)
 651 {
 652   snat_main_t *sm = &snat_main;
 653   u32 thread_idx = sm->num_workers;
 654   if (sm->num_workers > 1)
 655     {
 656       thread_idx =
 657         sm->first_worker_index +
 658         sm->workers[(e_port - 1024) / sm->port_per_thread];
 659     }
 660   return thread_idx;
 661 }
 662
 663 void
 664 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 665                                     snat_main_per_thread_data_t * tsm,
 666                                     ip4_address_t l_addr,
 667                                     u16 l_port,
 668                                     u8 protocol,
 669                                     u32 fib_index, int addr_only,
 670                                     ip4_address_t e_addr, u16 e_port)
 671 {
 672   snat_session_t *s;
 673   u32 *indexes_to_free = NULL;
 674   /* *INDENT-OFF* */
 675   pool_foreach (s, tsm->sessions) {
 676     if (s->in2out.fib_index != fib_index ||
 677         s->in2out.addr.as_u32 != l_addr.as_u32)
 678       {
 679         continue;
 680       }
 681     if (!addr_only)
 682       {
 683         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 684             s->out2in.port != e_port ||
 685             s->in2out.port != l_port ||
 686             s->nat_proto != protocol)
 687           continue;
 688       }
 689
 690     if (is_lb_session (s))
 691       continue;
 692     if (!snat_is_session_static (s))
 693       continue;
 694     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 695     vec_add1 (indexes_to_free, s - tsm->sessions);
 696     if (!addr_only)
 697       break;
 698   }
 699   /* *INDENT-ON* */
 700   u32 *ses_index;
 701   vec_foreach (ses_index, indexes_to_free)
 702   {
 703     s = pool_elt_at_index (tsm->sessions, *ses_index);
 704     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 705   }
 706   vec_free (indexes_to_free);
 707 }
 708
 709 static_always_inline int
 710 nat44_ed_add_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 711                                  u16 l_port, u16 e_port, u32 vrf_id,
 712                                  int addr_only, u32 sw_if_index,
 713                                  nat_protocol_t proto, int is_add,
 714                                  twice_nat_type_t twice_nat, u8 out2in_only,
 715                                  u8 *tag, u8 identity_nat,
 716                                  ip4_address_t pool_addr, int exact)
 717 {
 718   snat_main_t *sm = &snat_main;
 719   snat_static_mapping_t *m;
 720   clib_bihash_kv_8_8_t kv, value;
 721   snat_address_t *a = 0;
 722   u32 fib_index = ~0;
 723   snat_interface_t *interface;
 724   int i;
 725   snat_main_per_thread_data_t *tsm;
 726   snat_user_key_t u_key;
 727   snat_user_t *u;
 728   dlist_elt_t *head, *elt;
 729   u32 elt_index, head_index;
 730   u32 ses_index;
 731   u64 user_index;
 732   snat_session_t *s;
 733   snat_static_map_resolve_t *rp, *rp_match = 0;
 734   nat44_lb_addr_port_t *local;
 735   u32 find = ~0;
 736
 737   if (!sm->endpoint_dependent)
 738     {
 739       if (twice_nat || out2in_only)
 740         return VNET_API_ERROR_UNSUPPORTED;
 741     }
 742
 743   /* If the external address is a specific interface address */
 744   if (sw_if_index != ~0)
 745     {
 746       ip4_address_t *first_int_addr;
 747
 748       for (i = 0; i < vec_len (sm->to_resolve); i++)
 749         {
 750           rp = sm->to_resolve + i;
 751           if (rp->sw_if_index != sw_if_index ||
 752               rp->l_addr.as_u32 != l_addr.as_u32 ||
 753               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 754             continue;
 755
 756           if (!addr_only)
 757             {
 758               if ((rp->l_port != l_port && rp->e_port != e_port)
 759                   || rp->proto != proto)
 760                 continue;
 761             }
 762
 763           rp_match = rp;
 764           break;
 765         }
 766
 767       /* Might be already set... */
 768       first_int_addr = ip4_interface_first_address
 769         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 770
 771       if (is_add)
 772         {
 773           if (rp_match)
 774             return VNET_API_ERROR_VALUE_EXIST;
 775
 776           snat_add_static_mapping_when_resolved (
 777             sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto, addr_only,
 778             tag, twice_nat, out2in_only, identity_nat, pool_addr, exact);
 779
 780           /* DHCP resolution required? */
 781           if (first_int_addr == 0)
 782             {
 783               return 0;
 784             }
 785           else
 786             {
 787               e_addr.as_u32 = first_int_addr->as_u32;
 788               /* Identity mapping? */
 789               if (l_addr.as_u32 == 0)
 790                 l_addr.as_u32 = e_addr.as_u32;
 791             }
 792         }
 793       else
 794         {
 795           if (!rp_match)
 796             return VNET_API_ERROR_NO_SUCH_ENTRY;
 797
 798           vec_del1 (sm->to_resolve, i);
 799
 800           if (first_int_addr)
 801             {
 802               e_addr.as_u32 = first_int_addr->as_u32;
 803               /* Identity mapping? */
 804               if (l_addr.as_u32 == 0)
 805                 l_addr.as_u32 = e_addr.as_u32;
 806             }
 807           else
 808             return 0;
 809         }
 810     }
 811
 812   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 813   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 814     m = 0;
 815   else
 816     m = pool_elt_at_index (sm->static_mappings, value.value);
 817
 818   if (is_add)
 819     {
 820       if (m)
 821         {
 822           if (is_identity_static_mapping (m))
 823             {
 824               /* *INDENT-OFF* */
 825               pool_foreach (local, m->locals)
 826                {
 827                 if (local->vrf_id == vrf_id)
 828                   return VNET_API_ERROR_VALUE_EXIST;
 829               }
 830               /* *INDENT-ON* */
 831               pool_get (m->locals, local);
 832               local->vrf_id = vrf_id;
 833               local->fib_index =
 834                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 835                                                    sm->fib_src_low);
 836               init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
 837                            m->proto, 0, m - sm->static_mappings);
 838               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 839               return 0;
 840             }
 841           else
 842             return VNET_API_ERROR_VALUE_EXIST;
 843         }
 844
 845       if (twice_nat && addr_only)
 846         return VNET_API_ERROR_UNSUPPORTED;
 847
 848       /* Convert VRF id to FIB index */
 849       if (vrf_id != ~0)
 850         fib_index =
 851           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 852                                              sm->fib_src_low);
 853       /* If not specified use inside VRF id from SNAT plugin startup config */
 854       else
 855         {
 856           fib_index = sm->inside_fib_index;
 857           vrf_id = sm->inside_vrf_id;
 858           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 859         }
 860
 861       if (!(out2in_only || identity_nat))
 862         {
 863           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
 864                       addr_only ? 0 : proto);
 865           if (!clib_bihash_search_8_8
 866               (&sm->static_mapping_by_local, &kv, &value))
 867             return VNET_API_ERROR_VALUE_EXIST;
 868         }
 869
 870       /* Find external address in allocated addresses and reserve port for
 871          address and port pair mapping when dynamic translations enabled */
 872       if (!(addr_only || sm->static_mapping_only || out2in_only))
 873         {
 874           for (i = 0; i < vec_len (sm->addresses); i++)
 875             {
 876               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 877                 {
 878                   a = sm->addresses + i;
 879                   /* External port must be unused */
 880                   switch (proto)
 881                     {
 882 #define _(N, j, n, s) \
 883                     case NAT_PROTOCOL_##N: \
 884                       if (a->busy_##n##_port_refcounts[e_port]) \
 885                         return VNET_API_ERROR_INVALID_VALUE; \
 886                       ++a->busy_##n##_port_refcounts[e_port]; \
 887                       if (e_port > 1024) \
 888                         { \
 889                           a->busy_##n##_ports++; \
 890                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 891                         } \
 892                       break;
 893                       foreach_nat_protocol
 894 #undef _
 895                     default:
 896                       nat_elog_info ("unknown protocol");
 897                       return VNET_API_ERROR_INVALID_VALUE_2;
 898                     }
 899                   break;
 900                 }
 901             }
 902           /* External address must be allocated */
 903           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 904             {
 905               if (sw_if_index != ~0)
 906                 {
 907                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 908                     {
 909                       rp = sm->to_resolve + i;
 910                       if (rp->addr_only)
 911                         continue;
 912                       if (rp->sw_if_index != sw_if_index &&
 913                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 914                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 915                           rp->e_port != e_port && rp->proto != proto)
 916                         continue;
 917
 918                       vec_del1 (sm->to_resolve, i);
 919                       break;
 920                     }
 921                 }
 922               return VNET_API_ERROR_NO_SUCH_ENTRY;
 923             }
 924         }
 925
 926       pool_get (sm->static_mappings, m);
 927       clib_memset (m, 0, sizeof (*m));
 928       m->tag = vec_dup (tag);
 929       m->local_addr = l_addr;
 930       m->external_addr = e_addr;
 931       m->twice_nat = twice_nat;
 932
 933       if (twice_nat == TWICE_NAT && exact)
 934         {
 935           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
 936           m->pool_addr = pool_addr;
 937         }
 938
 939       if (out2in_only)
 940         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 941       if (addr_only)
 942         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 943       if (identity_nat)
 944         {
 945           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 946           pool_get (m->locals, local);
 947           local->vrf_id = vrf_id;
 948           local->fib_index = fib_index;
 949         }
 950       else
 951         {
 952           m->vrf_id = vrf_id;
 953           m->fib_index = fib_index;
 954         }
 955       if (!addr_only)
 956         {
 957           m->local_port = l_port;
 958           m->external_port = e_port;
 959           m->proto = proto;
 960         }
 961
 962       if (sm->num_workers > 1)
 963         {
 964           ip4_header_t ip = {
 965             .src_address = m->local_addr,
 966           };
 967           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 968           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 969         }
 970       else
 971         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 972
 973       if (!out2in_only)
 974         {
 975           init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
 976                        0, m - sm->static_mappings);
 977           clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 978         }
 979
 980       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
 981                    m - sm->static_mappings);
 982       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 983
 984       /* Delete dynamic sessions matching local address (+ local port) */
 985       // TODO: based on type of NAT EI/ED
 986       if (!(sm->static_mapping_only))
 987         {
 988           u_key.addr = m->local_addr;
 989           u_key.fib_index = m->fib_index;
 990           kv.key = u_key.as_u64;
 991           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 992             {
 993               user_index = value.value;
 994               u = pool_elt_at_index (tsm->users, user_index);
 995               if (u->nsessions)
 996                 {
 997                   head_index = u->sessions_per_user_list_head_index;
 998                   head = pool_elt_at_index (tsm->list_pool, head_index);
 999                   elt_index = head->next;
1000                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1001                   ses_index = elt->value;
1002                   while (ses_index != ~0)
1003                     {
1004                       s = pool_elt_at_index (tsm->sessions, ses_index);
1005                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1006                       ses_index = elt->value;
1007
1008                       if (snat_is_session_static (s))
1009                         continue;
1010
1011                       if (!addr_only && s->in2out.port != m->local_port)
1012                         continue;
1013
1014                       nat_free_session_data (sm, s,
1015                                              tsm - sm->per_thread_data, 0);
1016                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1017
1018                       if (!addr_only && !sm->endpoint_dependent)
1019                         break;
1020                     }
1021                 }
1022             }
1023         }
1024     }
1025   else
1026     {
1027       if (!m)
1028         {
1029           if (sw_if_index != ~0)
1030             return 0;
1031           else
1032             return VNET_API_ERROR_NO_SUCH_ENTRY;
1033         }
1034
1035       if (identity_nat)
1036         {
1037           if (vrf_id == ~0)
1038             vrf_id = sm->inside_vrf_id;
1039
1040           /* *INDENT-OFF* */
1041           pool_foreach (local, m->locals)
1042            {
1043             if (local->vrf_id == vrf_id)
1044               find = local - m->locals;
1045           }
1046           /* *INDENT-ON* */
1047           if (find == ~0)
1048             return VNET_API_ERROR_NO_SUCH_ENTRY;
1049
1050           local = pool_elt_at_index (m->locals, find);
1051           fib_index = local->fib_index;
1052           pool_put (m->locals, local);
1053         }
1054       else
1055         fib_index = m->fib_index;
1056
1057       /* Free external address port */
1058       if (!(addr_only || sm->static_mapping_only || out2in_only))
1059         {
1060           for (i = 0; i < vec_len (sm->addresses); i++)
1061             {
1062               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1063                 {
1064                   a = sm->addresses + i;
1065                   switch (proto)
1066                     {
1067 #define _(N, j, n, s) \
1068                     case NAT_PROTOCOL_##N: \
1069                       --a->busy_##n##_port_refcounts[e_port]; \
1070                       if (e_port > 1024) \
1071                         { \
1072                           a->busy_##n##_ports--; \
1073                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1074                         } \
1075                       break;
1076                       foreach_nat_protocol
1077 #undef _
1078                     default:
1079                       nat_elog_info ("unknown protocol");
1080                       return VNET_API_ERROR_INVALID_VALUE_2;
1081                     }
1082                   break;
1083                 }
1084             }
1085         }
1086
1087       if (sm->num_workers > 1)
1088         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1089       else
1090         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1091
1092       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1093       if (!out2in_only)
1094         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1095
1096       /* Delete session(s) for static mapping if exist */
1097       if (!(sm->static_mapping_only) ||
1098           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1099         {
1100           if (sm->endpoint_dependent)
1101             {
1102               nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1103                                                   m->local_port, m->proto,
1104                                                   fib_index, addr_only,
1105                                                   e_addr, e_port);
1106             }
1107           else
1108             {
1109               u_key.addr = m->local_addr;
1110               u_key.fib_index = fib_index;
1111               kv.key = u_key.as_u64;
1112               nat44_ei_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1113                                                     e_addr, e_port);
1114             }
1115         }
1116
1117       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1118       if (pool_elts (m->locals))
1119         return 0;
1120
1121       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1122       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1123
1124       vec_free (m->tag);
1125       vec_free (m->workers);
1126       /* Delete static mapping from pool */
1127       pool_put (sm->static_mappings, m);
1128     }
1129
1130   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1131     return 0;
1132
1133   /* Add/delete external address to FIB */
1134   /* *INDENT-OFF* */
1135   pool_foreach (interface, sm->interfaces)
1136    {
1137     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1138       continue;
1139
1140     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1141     break;
1142   }
1143   pool_foreach (interface, sm->output_feature_interfaces)
1144    {
1145     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1146       continue;
1147
1148     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1149     break;
1150   }
1151   /* *INDENT-ON* */
1152
1153   return 0;
1154 }
1155
1156 int
1157 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
1158                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
1159                          u32 sw_if_index, nat_protocol_t proto, int is_add,
1160                          twice_nat_type_t twice_nat, u8 out2in_only, u8 *tag,
1161                          u8 identity_nat, ip4_address_t pool_addr, int exact)
1162 {
1163   snat_main_t *sm = &snat_main;
1164   int rv;
1165
1166   if (sm->endpoint_dependent)
1167     {
1168       rv = nat44_ed_add_del_static_mapping (
1169         l_addr, e_addr, l_port, e_port, vrf_id, addr_only, sw_if_index, proto,
1170         is_add, twice_nat, out2in_only, tag, identity_nat, pool_addr, exact);
1171     }
1172   else
1173     {
1174       rv = nat44_ei_add_del_static_mapping (
1175         l_addr, e_addr, l_port, e_port, proto, sw_if_index, vrf_id, addr_only,
1176         identity_nat, tag, is_add);
1177     }
1178   return rv;
1179 }
1180
1181 int
1182 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1183                                  nat_protocol_t proto,
1184                                  nat44_lb_addr_port_t * locals, u8 is_add,
1185                                  twice_nat_type_t twice_nat, u8 out2in_only,
1186                                  u8 * tag, u32 affinity)
1187 {
1188   snat_main_t *sm = &snat_main;
1189   snat_static_mapping_t *m;
1190   clib_bihash_kv_8_8_t kv, value;
1191   snat_address_t *a = 0;
1192   int i;
1193   nat44_lb_addr_port_t *local;
1194   snat_main_per_thread_data_t *tsm;
1195   snat_session_t *s;
1196   uword *bitmap = 0;
1197
1198   if (!sm->endpoint_dependent)
1199     return VNET_API_ERROR_UNSUPPORTED;
1200
1201   init_nat_k (&kv, e_addr, e_port, 0, proto);
1202   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1203     m = 0;
1204   else
1205     m = pool_elt_at_index (sm->static_mappings, value.value);
1206
1207   if (is_add)
1208     {
1209       if (m)
1210         return VNET_API_ERROR_VALUE_EXIST;
1211
1212       if (vec_len (locals) < 2)
1213         return VNET_API_ERROR_INVALID_VALUE;
1214
1215       /* Find external address in allocated addresses and reserve port for
1216          address and port pair mapping when dynamic translations enabled */
1217       if (!(sm->static_mapping_only || out2in_only))
1218         {
1219           for (i = 0; i < vec_len (sm->addresses); i++)
1220             {
1221               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1222                 {
1223                   a = sm->addresses + i;
1224                   /* External port must be unused */
1225                   switch (proto)
1226                     {
1227 #define _(N, j, n, s) \
1228                     case NAT_PROTOCOL_##N: \
1229                       if (a->busy_##n##_port_refcounts[e_port]) \
1230                         return VNET_API_ERROR_INVALID_VALUE; \
1231                       ++a->busy_##n##_port_refcounts[e_port]; \
1232                       if (e_port > 1024) \
1233                         { \
1234                           a->busy_##n##_ports++; \
1235                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1236                         } \
1237                       break;
1238                       foreach_nat_protocol
1239 #undef _
1240                     default:
1241                       nat_elog_info ("unknown protocol");
1242                       return VNET_API_ERROR_INVALID_VALUE_2;
1243                     }
1244                   break;
1245                 }
1246             }
1247           /* External address must be allocated */
1248           if (!a)
1249             return VNET_API_ERROR_NO_SUCH_ENTRY;
1250         }
1251
1252       pool_get (sm->static_mappings, m);
1253       clib_memset (m, 0, sizeof (*m));
1254       m->tag = vec_dup (tag);
1255       m->external_addr = e_addr;
1256       m->external_port = e_port;
1257       m->proto = proto;
1258       m->twice_nat = twice_nat;
1259       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1260       if (out2in_only)
1261         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1262       m->affinity = affinity;
1263
1264       if (affinity)
1265         m->affinity_per_service_list_head_index =
1266           nat_affinity_get_per_service_list_head_index ();
1267       else
1268         m->affinity_per_service_list_head_index = ~0;
1269
1270       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
1271                    m - sm->static_mappings);
1272       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1273         {
1274           nat_elog_err ("static_mapping_by_external key add failed");
1275           return VNET_API_ERROR_UNSPECIFIED;
1276         }
1277
1278       for (i = 0; i < vec_len (locals); i++)
1279         {
1280           locals[i].fib_index =
1281             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1282                                                locals[i].vrf_id,
1283                                                sm->fib_src_low);
1284           if (!out2in_only)
1285             {
1286               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1287                            locals[i].fib_index, m->proto, 0,
1288                            m - sm->static_mappings);
1289               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1290             }
1291           locals[i].prefix = (i == 0) ? locals[i].probability :
1292             (locals[i - 1].prefix + locals[i].probability);
1293           pool_get (m->locals, local);
1294           *local = locals[i];
1295           if (sm->num_workers > 1)
1296             {
1297               ip4_header_t ip = {
1298                 .src_address = locals[i].addr,
1299               };
1300               bitmap =
1301                 clib_bitmap_set (bitmap,
1302                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1303                                  1);
1304             }
1305         }
1306
1307       /* Assign workers */
1308       if (sm->num_workers > 1)
1309         {
1310           /* *INDENT-OFF* */
1311           clib_bitmap_foreach (i, bitmap)
1312              {
1313                vec_add1(m->workers, i);
1314             }
1315           /* *INDENT-ON* */
1316         }
1317     }
1318   else
1319     {
1320       if (!m)
1321         return VNET_API_ERROR_NO_SUCH_ENTRY;
1322
1323       if (!is_lb_static_mapping (m))
1324         return VNET_API_ERROR_INVALID_VALUE;
1325
1326       /* Free external address port */
1327       if (!(sm->static_mapping_only || out2in_only))
1328         {
1329           for (i = 0; i < vec_len (sm->addresses); i++)
1330             {
1331               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1332                 {
1333                   a = sm->addresses + i;
1334                   switch (proto)
1335                     {
1336 #define _(N, j, n, s) \
1337                     case NAT_PROTOCOL_##N: \
1338                       --a->busy_##n##_port_refcounts[e_port]; \
1339                       if (e_port > 1024) \
1340                         { \
1341                           a->busy_##n##_ports--; \
1342                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1343                         } \
1344                       break;
1345                       foreach_nat_protocol
1346 #undef _
1347                     default:
1348                       nat_elog_info ("unknown protocol");
1349                       return VNET_API_ERROR_INVALID_VALUE_2;
1350                     }
1351                   break;
1352                 }
1353             }
1354         }
1355
1356       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1357       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1358         {
1359           nat_elog_err ("static_mapping_by_external key del failed");
1360           return VNET_API_ERROR_UNSPECIFIED;
1361         }
1362
1363       /* *INDENT-OFF* */
1364       pool_foreach (local, m->locals)
1365       {
1366           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1367                             sm->fib_src_low);
1368           if (!out2in_only)
1369             {
1370 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1371               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1372                 {
1373                   nat_elog_err ("static_mapping_by_local key del failed");
1374                   return VNET_API_ERROR_UNSPECIFIED;
1375                 }
1376             }
1377
1378           if (sm->num_workers > 1)
1379             {
1380               ip4_header_t ip = {
1381                 .src_address = local->addr,
1382               };
1383               tsm = vec_elt_at_index (sm->per_thread_data,
1384                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1385             }
1386           else
1387             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1388
1389           /* Delete sessions */
1390           pool_foreach (s, tsm->sessions) {
1391             if (!(is_lb_session (s)))
1392               continue;
1393
1394             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1395                 s->in2out.port != local->port)
1396               continue;
1397
1398             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1399             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1400           }
1401       }
1402       /* *INDENT-ON* */
1403       if (m->affinity)
1404         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1405       pool_free (m->locals);
1406       vec_free (m->tag);
1407       vec_free (m->workers);
1408
1409       pool_put (sm->static_mappings, m);
1410     }
1411
1412   return 0;
1413 }
1414
1415 int
1416 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1417                                        ip4_address_t l_addr, u16 l_port,
1418                                        nat_protocol_t proto, u32 vrf_id,
1419                                        u8 probability, u8 is_add)
1420 {
1421   snat_main_t *sm = &snat_main;
1422   snat_static_mapping_t *m = 0;
1423   clib_bihash_kv_8_8_t kv, value;
1424   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1425   snat_main_per_thread_data_t *tsm;
1426   snat_session_t *s;
1427   u32 *locals = 0;
1428   uword *bitmap = 0;
1429   int i;
1430
1431   if (!sm->endpoint_dependent)
1432     return VNET_API_ERROR_FEATURE_DISABLED;
1433
1434   init_nat_k (&kv, e_addr, e_port, 0, proto);
1435   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1436     m = pool_elt_at_index (sm->static_mappings, value.value);
1437
1438   if (!m)
1439     return VNET_API_ERROR_NO_SUCH_ENTRY;
1440
1441   if (!is_lb_static_mapping (m))
1442     return VNET_API_ERROR_INVALID_VALUE;
1443
1444   /* *INDENT-OFF* */
1445   pool_foreach (local, m->locals)
1446    {
1447     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1448         (local->vrf_id == vrf_id))
1449       {
1450         match_local = local;
1451         break;
1452       }
1453   }
1454   /* *INDENT-ON* */
1455
1456   if (is_add)
1457     {
1458       if (match_local)
1459         return VNET_API_ERROR_VALUE_EXIST;
1460
1461       pool_get (m->locals, local);
1462       clib_memset (local, 0, sizeof (*local));
1463       local->addr.as_u32 = l_addr.as_u32;
1464       local->port = l_port;
1465       local->probability = probability;
1466       local->vrf_id = vrf_id;
1467       local->fib_index =
1468         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1469                                            sm->fib_src_low);
1470
1471       if (!is_out2in_only_static_mapping (m))
1472         {
1473           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto, 0,
1474                        m - sm->static_mappings);
1475           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1476             nat_elog_err ("static_mapping_by_local key add failed");
1477         }
1478     }
1479   else
1480     {
1481       if (!match_local)
1482         return VNET_API_ERROR_NO_SUCH_ENTRY;
1483
1484       if (pool_elts (m->locals) < 3)
1485         return VNET_API_ERROR_UNSPECIFIED;
1486
1487       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1488                         sm->fib_src_low);
1489
1490       if (!is_out2in_only_static_mapping (m))
1491         {
1492           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1493           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1494             nat_elog_err ("static_mapping_by_local key del failed");
1495         }
1496
1497       if (sm->num_workers > 1)
1498         {
1499           ip4_header_t ip = {
1500             .src_address = local->addr,
1501           };
1502           tsm = vec_elt_at_index (sm->per_thread_data,
1503                                   sm->worker_in2out_cb (&ip, m->fib_index,
1504                                                         0));
1505         }
1506       else
1507         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1508
1509       /* Delete sessions */
1510       /* *INDENT-OFF* */
1511       pool_foreach (s, tsm->sessions) {
1512         if (!(is_lb_session (s)))
1513           continue;
1514
1515         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1516             s->in2out.port != match_local->port)
1517           continue;
1518
1519         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1520         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1521       }
1522       /* *INDENT-ON* */
1523
1524       pool_put (m->locals, match_local);
1525     }
1526
1527   vec_free (m->workers);
1528
1529   /* *INDENT-OFF* */
1530   pool_foreach (local, m->locals)
1531    {
1532     vec_add1 (locals, local - m->locals);
1533     if (sm->num_workers > 1)
1534       {
1535         ip4_header_t ip;
1536         ip.src_address.as_u32 = local->addr.as_u32,
1537         bitmap = clib_bitmap_set (bitmap,
1538                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1539                                   1);
1540       }
1541   }
1542   /* *INDENT-ON* */
1543
1544   ASSERT (vec_len (locals) > 1);
1545
1546   local = pool_elt_at_index (m->locals, locals[0]);
1547   local->prefix = local->probability;
1548   for (i = 1; i < vec_len (locals); i++)
1549     {
1550       local = pool_elt_at_index (m->locals, locals[i]);
1551       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1552       local->prefix = local->probability + prev_local->prefix;
1553     }
1554
1555   /* Assign workers */
1556   if (sm->num_workers > 1)
1557     {
1558       /* *INDENT-OFF* */
1559       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1560       /* *INDENT-ON* */
1561     }
1562
1563   return 0;
1564 }
1565
1566 int
1567 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1568                   u8 twice_nat)
1569 {
1570   snat_address_t *a = 0;
1571   snat_session_t *ses;
1572   u32 *ses_to_be_removed = 0, *ses_index;
1573   snat_main_per_thread_data_t *tsm;
1574   snat_static_mapping_t *m;
1575   snat_interface_t *interface;
1576   int i;
1577   snat_address_t *addresses =
1578     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1579
1580   /* Find SNAT address */
1581   for (i = 0; i < vec_len (addresses); i++)
1582     {
1583       if (addresses[i].addr.as_u32 == addr.as_u32)
1584         {
1585           a = addresses + i;
1586           break;
1587         }
1588     }
1589   if (!a)
1590     {
1591       nat_log_err ("no such address");
1592       return VNET_API_ERROR_NO_SUCH_ENTRY;
1593     }
1594
1595   if (delete_sm)
1596     {
1597       ip4_address_t pool_addr = { 0 };
1598       /* *INDENT-OFF* */
1599       pool_foreach (m, sm->static_mappings)
1600        {
1601           if (m->external_addr.as_u32 == addr.as_u32)
1602             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1603                                             m->local_port, m->external_port,
1604                                             m->vrf_id,
1605                                             is_addr_only_static_mapping(m), ~0,
1606                                             m->proto, 0 /* is_add */,
1607                                             m->twice_nat,
1608                                             is_out2in_only_static_mapping(m),
1609                                             m->tag,
1610                                             is_identity_static_mapping(m),
1611                                             pool_addr, 0);
1612       }
1613       /* *INDENT-ON* */
1614     }
1615   else
1616     {
1617       /* Check if address is used in some static mapping */
1618       if (is_snat_address_used_in_static_mapping (sm, addr))
1619         {
1620           nat_log_err ("address used in static mapping");
1621           return VNET_API_ERROR_UNSPECIFIED;
1622         }
1623     }
1624
1625   if (a->fib_index != ~0)
1626     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1627
1628   /* Delete sessions using address */
1629   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1630     {
1631       vec_foreach (tsm, sm->per_thread_data)
1632       {
1633         /* *INDENT-OFF* */
1634         pool_foreach (ses, tsm->sessions)  {
1635           if (ses->out2in.addr.as_u32 == addr.as_u32)
1636             {
1637               nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1638               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1639             }
1640         }
1641         /* *INDENT-ON* */
1642
1643         if (sm->endpoint_dependent)
1644           {
1645             vec_foreach (ses_index, ses_to_be_removed)
1646             {
1647               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1648               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1649             }
1650           }
1651         else
1652           {
1653             vec_foreach (ses_index, ses_to_be_removed)
1654             {
1655               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1656               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1657             }
1658           }
1659
1660         vec_free (ses_to_be_removed);
1661       }
1662     }
1663
1664 #define _(N, i, n, s) \
1665   vec_free (a->busy_##n##_ports_per_thread);
1666   foreach_nat_protocol
1667 #undef _
1668
1669     if (twice_nat)
1670   {
1671     vec_del1 (sm->twice_nat_addresses, i);
1672     return 0;
1673   }
1674   else vec_del1 (sm->addresses, i);
1675
1676   /* Delete external address from FIB */
1677   pool_foreach (interface, sm->interfaces)
1678     {
1679       if (nat_interface_is_inside (interface) || sm->out2in_dpo)
1680         continue;
1681
1682       snat_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
1683       break;
1684     }
1685
1686   pool_foreach (interface, sm->output_feature_interfaces)
1687    {
1688     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1689       continue;
1690
1691     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1692     break;
1693   }
1694
1695   return 0;
1696 }
1697
1698 static void
1699 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1700 {
1701 #define _(x)                                                                  \
1702   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1703                                 sw_if_index);                                 \
1704   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1705   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1706                                 sw_if_index);                                 \
1707   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1708   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1709                                 sw_if_index);                                 \
1710   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1711   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1712                                 sw_if_index);                                 \
1713   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1714   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1715                                 sw_if_index);                                 \
1716   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1717   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1718                                 sw_if_index);                                 \
1719   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1720   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1721                                 sw_if_index);                                 \
1722   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1723   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1724                                 sw_if_index);                                 \
1725   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1726   foreach_nat_counter;
1727 #undef _
1728   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1729   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1730 }
1731
1732 void
1733 expire_per_vrf_sessions (u32 fib_index)
1734 {
1735   per_vrf_sessions_t *per_vrf_sessions;
1736   snat_main_per_thread_data_t *tsm;
1737   snat_main_t *sm = &snat_main;
1738
1739   /* *INDENT-OFF* */
1740   vec_foreach (tsm, sm->per_thread_data)
1741     {
1742       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1743         {
1744           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1745               (per_vrf_sessions->tx_fib_index == fib_index))
1746             {
1747               per_vrf_sessions->expired = 1;
1748             }
1749         }
1750     }
1751   /* *INDENT-ON* */
1752 }
1753
1754 void
1755 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1756 {
1757   snat_main_t *sm = &snat_main;
1758   nat_fib_t *fib;
1759
1760   // we don't care if it is outside/inside fib
1761   // we just care about their ref_count
1762   // if it reaches 0 sessions should expire
1763   // because the fib isn't valid for NAT anymore
1764
1765   vec_foreach (fib, sm->fibs)
1766   {
1767     if (fib->fib_index == fib_index)
1768       {
1769         if (is_del)
1770           {
1771             fib->ref_count--;
1772             if (!fib->ref_count)
1773               {
1774                 vec_del1 (sm->fibs, fib - sm->fibs);
1775                 expire_per_vrf_sessions (fib_index);
1776               }
1777             return;
1778           }
1779         else
1780           fib->ref_count++;
1781       }
1782   }
1783   if (!is_del)
1784     {
1785       vec_add2 (sm->fibs, fib, 1);
1786       fib->ref_count = 1;
1787       fib->fib_index = fib_index;
1788     }
1789 }
1790
1791 int
1792 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1793 {
1794   snat_main_t *sm = &snat_main;
1795   snat_interface_t *i;
1796   const char *feature_name, *del_feature_name;
1797   snat_address_t *ap;
1798   snat_static_mapping_t *m;
1799   nat_outside_fib_t *outside_fib;
1800   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1801                                                        sw_if_index);
1802
1803   if (!sm->enabled)
1804     {
1805       nat_log_err ("nat44 is disabled");
1806       return VNET_API_ERROR_UNSUPPORTED;
1807     }
1808
1809   if (sm->out2in_dpo && !is_inside)
1810     {
1811       nat_log_err ("error unsupported");
1812       return VNET_API_ERROR_UNSUPPORTED;
1813     }
1814
1815   /* *INDENT-OFF* */
1816   pool_foreach (i, sm->output_feature_interfaces)
1817    {
1818     if (i->sw_if_index == sw_if_index)
1819       {
1820         nat_log_err ("error interface already configured");
1821         return VNET_API_ERROR_VALUE_EXIST;
1822       }
1823   }
1824   /* *INDENT-ON* */
1825
1826   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1827     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1828   else
1829     {
1830       if (sm->num_workers > 1)
1831         feature_name =
1832           is_inside ? "nat44-in2out-worker-handoff" :
1833           "nat44-out2in-worker-handoff";
1834       else if (sm->endpoint_dependent)
1835         {
1836           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1837         }
1838       else
1839         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1840     }
1841
1842   ASSERT (sm->frame_queue_nelts > 0);
1843
1844   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
1845     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1846                                                       sm->frame_queue_nelts);
1847
1848   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1849     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1850                                                       sm->frame_queue_nelts);
1851
1852   if (sm->endpoint_dependent)
1853     update_per_vrf_sessions_vec (fib_index, is_del);
1854
1855   if (!is_inside)
1856     {
1857       /* *INDENT-OFF* */
1858       vec_foreach (outside_fib, sm->outside_fibs)
1859         {
1860           if (outside_fib->fib_index == fib_index)
1861             {
1862               if (is_del)
1863                 {
1864                   outside_fib->refcount--;
1865                   if (!outside_fib->refcount)
1866                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1867                 }
1868               else
1869                 outside_fib->refcount++;
1870               goto feature_set;
1871             }
1872         }
1873       /* *INDENT-ON* */
1874       if (!is_del)
1875         {
1876           vec_add2 (sm->outside_fibs, outside_fib, 1);
1877           outside_fib->refcount = 1;
1878           outside_fib->fib_index = fib_index;
1879         }
1880     }
1881
1882 feature_set:
1883   /* *INDENT-OFF* */
1884   pool_foreach (i, sm->interfaces)
1885    {
1886     if (i->sw_if_index == sw_if_index)
1887       {
1888         if (is_del)
1889           {
1890             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1891               {
1892                 if (is_inside)
1893                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1894                 else
1895                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1896
1897                 if (sm->num_workers > 1)
1898                   {
1899                     del_feature_name = "nat44-handoff-classify";
1900                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1901                                                  "nat44-out2in-worker-handoff";
1902                   }
1903                 else if (sm->endpoint_dependent)
1904                   {
1905                     del_feature_name = "nat44-ed-classify";
1906                     feature_name = !is_inside ?  "nat-pre-in2out" :
1907                                                  "nat-pre-out2in";
1908                   }
1909                 else
1910                   {
1911                     del_feature_name = "nat44-classify";
1912                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1913                   }
1914
1915                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1916                 if (rv)
1917                   return rv;
1918                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1919                                              sw_if_index, 0, 0, 0);
1920                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1921                                              sw_if_index, 1, 0, 0);
1922                 if (!is_inside)
1923                   {
1924                     if (!sm->endpoint_dependent)
1925                       vnet_feature_enable_disable ("ip4-local",
1926                                                    "nat44-hairpinning",
1927                                                    sw_if_index, 1, 0, 0);
1928                   }
1929               }
1930             else
1931               {
1932                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1933                 if (rv)
1934                   return rv;
1935                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1936                                              sw_if_index, 0, 0, 0);
1937                 pool_put (sm->interfaces, i);
1938                 if (is_inside)
1939                   {
1940                     if (!sm->endpoint_dependent)
1941                       vnet_feature_enable_disable ("ip4-local",
1942                                                    "nat44-hairpinning",
1943                                                    sw_if_index, 0, 0, 0);
1944                   }
1945               }
1946           }
1947         else
1948           {
1949             if ((nat_interface_is_inside (i) && is_inside) ||
1950                 (nat_interface_is_outside (i) && !is_inside))
1951               return 0;
1952
1953             if (sm->num_workers > 1)
1954               {
1955                 del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1956                                                 "nat44-out2in-worker-handoff";
1957                 feature_name = "nat44-handoff-classify";
1958               }
1959             else if (sm->endpoint_dependent)
1960               {
1961                 del_feature_name =
1962                   !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1963
1964                 feature_name = "nat44-ed-classify";
1965               }
1966             else
1967               {
1968                 del_feature_name =
1969                   !is_inside ? "nat44-in2out" : "nat44-out2in";
1970                 feature_name = "nat44-classify";
1971               }
1972
1973             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1974             if (rv)
1975               return rv;
1976             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1977                                          sw_if_index, 0, 0, 0);
1978             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1979                                          sw_if_index, 1, 0, 0);
1980             if (!is_inside)
1981               {
1982                 if (!sm->endpoint_dependent)
1983                   vnet_feature_enable_disable (
1984                     "ip4-local", "nat44-hairpinning", sw_if_index, 0, 0, 0);
1985               }
1986             goto set_flags;
1987           }
1988
1989         goto fib;
1990       }
1991   }
1992   /* *INDENT-ON* */
1993
1994   if (is_del)
1995     {
1996       nat_log_err ("error interface couldn't be found");
1997       return VNET_API_ERROR_NO_SUCH_ENTRY;
1998     }
1999
2000   pool_get (sm->interfaces, i);
2001   i->sw_if_index = sw_if_index;
2002   i->flags = 0;
2003   nat_validate_counters (sm, sw_if_index);
2004
2005   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2006                                0);
2007
2008   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2009   if (rv)
2010     return rv;
2011
2012   if (is_inside && !sm->out2in_dpo)
2013     {
2014       if (!sm->endpoint_dependent)
2015         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2016                                      sw_if_index, 1, 0, 0);
2017     }
2018
2019 set_flags:
2020   if (is_inside)
2021     {
2022       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2023       return 0;
2024     }
2025   else
2026     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2027
2028   /* Add/delete external addresses to FIB */
2029 fib:
2030   /* *INDENT-OFF* */
2031   vec_foreach (ap, sm->addresses)
2032     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2033
2034   pool_foreach (m, sm->static_mappings)
2035    {
2036     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2037       continue;
2038
2039     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2040   }
2041   /* *INDENT-ON* */
2042
2043   return 0;
2044 }
2045
2046 int
2047 snat_interface_add_del_output_feature (u32 sw_if_index,
2048                                        u8 is_inside, int is_del)
2049 {
2050   snat_main_t *sm = &snat_main;
2051   snat_interface_t *i;
2052   snat_address_t *ap;
2053   snat_static_mapping_t *m;
2054   nat_outside_fib_t *outside_fib;
2055   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2056                                                        sw_if_index);
2057
2058   if (!sm->enabled)
2059     {
2060       nat_log_err ("nat44 is disabled");
2061       return VNET_API_ERROR_UNSUPPORTED;
2062     }
2063
2064   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2065     {
2066       nat_log_err ("error unsupported");
2067       return VNET_API_ERROR_UNSUPPORTED;
2068     }
2069
2070   /* *INDENT-OFF* */
2071   pool_foreach (i, sm->interfaces)
2072    {
2073     if (i->sw_if_index == sw_if_index)
2074       {
2075         nat_log_err ("error interface already configured");
2076         return VNET_API_ERROR_VALUE_EXIST;
2077       }
2078   }
2079   /* *INDENT-ON* */
2080
2081   if (sm->endpoint_dependent)
2082     update_per_vrf_sessions_vec (fib_index, is_del);
2083
2084   if (!is_inside)
2085     {
2086       /* *INDENT-OFF* */
2087       vec_foreach (outside_fib, sm->outside_fibs)
2088         {
2089           if (outside_fib->fib_index == fib_index)
2090             {
2091               if (is_del)
2092                 {
2093                   outside_fib->refcount--;
2094                   if (!outside_fib->refcount)
2095                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2096                 }
2097               else
2098                 outside_fib->refcount++;
2099               goto feature_set;
2100             }
2101         }
2102       /* *INDENT-ON* */
2103       if (!is_del)
2104         {
2105           vec_add2 (sm->outside_fibs, outside_fib, 1);
2106           outside_fib->refcount = 1;
2107           outside_fib->fib_index = fib_index;
2108         }
2109     }
2110
2111 feature_set:
2112   if (is_inside)
2113     {
2114       if (sm->endpoint_dependent)
2115         {
2116           int rv =
2117             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2118           if (rv)
2119             return rv;
2120           rv =
2121             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2122                                                             !is_del);
2123           if (rv)
2124             return rv;
2125         }
2126       else
2127         {
2128           int rv =
2129             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2130           if (rv)
2131             return rv;
2132           rv =
2133             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2134                                                             !is_del);
2135           if (rv)
2136             return rv;
2137           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2138                                        sw_if_index, !is_del, 0, 0);
2139           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2140                                        sw_if_index, !is_del, 0, 0);
2141         }
2142       goto fq;
2143     }
2144
2145   if (sm->num_workers > 1)
2146     {
2147       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2148       if (rv)
2149         return rv;
2150       rv =
2151         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2152       if (rv)
2153         return rv;
2154       vnet_feature_enable_disable ("ip4-unicast",
2155                                    "nat44-out2in-worker-handoff",
2156                                    sw_if_index, !is_del, 0, 0);
2157       vnet_feature_enable_disable ("ip4-output",
2158                                    "nat44-in2out-output-worker-handoff",
2159                                    sw_if_index, !is_del, 0, 0);
2160     }
2161   else
2162     {
2163       if (sm->endpoint_dependent)
2164         {
2165           int rv =
2166             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2167           if (rv)
2168             return rv;
2169           rv =
2170             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2171                                                             !is_del);
2172           if (rv)
2173             return rv;
2174           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2175                                        sw_if_index, !is_del, 0, 0);
2176           vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2177                                        sw_if_index, !is_del, 0, 0);
2178         }
2179       else
2180         {
2181           int rv =
2182             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2183           if (rv)
2184             return rv;
2185           rv =
2186             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2187                                                             !is_del);
2188           if (rv)
2189             return rv;
2190           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2191                                        sw_if_index, !is_del, 0, 0);
2192           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2193                                        sw_if_index, !is_del, 0, 0);
2194         }
2195     }
2196
2197 fq:
2198   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2199     sm->fq_in2out_output_index =
2200       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2201
2202   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2203     sm->fq_out2in_index =
2204       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2205
2206   /* *INDENT-OFF* */
2207   pool_foreach (i, sm->output_feature_interfaces)
2208    {
2209     if (i->sw_if_index == sw_if_index)
2210       {
2211         if (is_del)
2212           pool_put (sm->output_feature_interfaces, i);
2213         else
2214           return VNET_API_ERROR_VALUE_EXIST;
2215
2216         goto fib;
2217       }
2218   }
2219   /* *INDENT-ON* */
2220
2221   if (is_del)
2222     {
2223       nat_log_err ("error interface couldn't be found");
2224       return VNET_API_ERROR_NO_SUCH_ENTRY;
2225     }
2226
2227   pool_get (sm->output_feature_interfaces, i);
2228   i->sw_if_index = sw_if_index;
2229   i->flags = 0;
2230   nat_validate_counters (sm, sw_if_index);
2231   if (is_inside)
2232     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2233   else
2234     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2235
2236   /* Add/delete external addresses to FIB */
2237 fib:
2238   if (is_inside)
2239     return 0;
2240
2241   /* *INDENT-OFF* */
2242   vec_foreach (ap, sm->addresses)
2243     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2244
2245   pool_foreach (m, sm->static_mappings)
2246    {
2247     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2248       continue;
2249
2250     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2251   }
2252   /* *INDENT-ON* */
2253
2254   return 0;
2255 }
2256
2257 int
2258 snat_set_workers (uword * bitmap)
2259 {
2260   snat_main_t *sm = &snat_main;
2261   int i, j = 0;
2262
2263   if (sm->num_workers < 2)
2264     return VNET_API_ERROR_FEATURE_DISABLED;
2265
2266   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2267     return VNET_API_ERROR_INVALID_WORKER;
2268
2269   vec_free (sm->workers);
2270   /* *INDENT-OFF* */
2271   clib_bitmap_foreach (i, bitmap)
2272     {
2273       vec_add1(sm->workers, i);
2274       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2275       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2276       j++;
2277     }
2278   /* *INDENT-ON* */
2279
2280   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2281
2282   return 0;
2283 }
2284
2285 int
2286 snat_set_frame_queue_nelts (u32 frame_queue_nelts)
2287 {
2288   fail_if_enabled ();
2289   snat_main_t *sm = &snat_main;
2290   sm->frame_queue_nelts = frame_queue_nelts;
2291   return 0;
2292 }
2293
2294 static void
2295 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2296                          u32 sw_if_index, u32 new_fib_index,
2297                          u32 old_fib_index)
2298 {
2299   snat_main_t *sm = &snat_main;
2300   nat_outside_fib_t *outside_fib;
2301   snat_interface_t *i;
2302   u8 is_add = 1;
2303   u8 match = 0;
2304
2305   if (!sm->enabled || (new_fib_index == old_fib_index)
2306       || (!vec_len (sm->outside_fibs)))
2307     {
2308       return;
2309     }
2310
2311   /* *INDENT-OFF* */
2312   pool_foreach (i, sm->interfaces)
2313      {
2314       if (i->sw_if_index == sw_if_index)
2315         {
2316           if (!(nat_interface_is_outside (i)))
2317             return;
2318           match = 1;
2319         }
2320     }
2321
2322   pool_foreach (i, sm->output_feature_interfaces)
2323      {
2324       if (i->sw_if_index == sw_if_index)
2325         {
2326           if (!(nat_interface_is_outside (i)))
2327             return;
2328           match = 1;
2329         }
2330     }
2331   /* *INDENT-ON* */
2332
2333   if (!match)
2334     return;
2335
2336   vec_foreach (outside_fib, sm->outside_fibs)
2337   {
2338     if (outside_fib->fib_index == old_fib_index)
2339       {
2340         outside_fib->refcount--;
2341         if (!outside_fib->refcount)
2342           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2343         break;
2344       }
2345   }
2346
2347   vec_foreach (outside_fib, sm->outside_fibs)
2348   {
2349     if (outside_fib->fib_index == new_fib_index)
2350       {
2351         outside_fib->refcount++;
2352         is_add = 0;
2353         break;
2354       }
2355   }
2356
2357   if (is_add)
2358     {
2359       vec_add2 (sm->outside_fibs, outside_fib, 1);
2360       outside_fib->refcount = 1;
2361       outside_fib->fib_index = new_fib_index;
2362     }
2363 }
2364
2365 static void
2366 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2367                          u32 sw_if_index, u32 new_fib_index,
2368                          u32 old_fib_index);
2369
2370 static void
2371 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2372                                        uword opaque,
2373                                        u32 sw_if_index,
2374                                        ip4_address_t * address,
2375                                        u32 address_length,
2376                                        u32 if_address_index, u32 is_delete);
2377
2378 static void
2379 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2380                                  uword opaque,
2381                                  u32 sw_if_index,
2382                                  ip4_address_t * address,
2383                                  u32 address_length,
2384                                  u32 if_address_index, u32 is_delete);
2385
2386 void
2387 test_key_calc_split ()
2388 {
2389   ip4_address_t l_addr;
2390   l_addr.as_u8[0] = 1;
2391   l_addr.as_u8[1] = 1;
2392   l_addr.as_u8[2] = 1;
2393   l_addr.as_u8[3] = 1;
2394   ip4_address_t r_addr;
2395   r_addr.as_u8[0] = 2;
2396   r_addr.as_u8[1] = 2;
2397   r_addr.as_u8[2] = 2;
2398   r_addr.as_u8[3] = 2;
2399   u16 l_port = 40001;
2400   u16 r_port = 40301;
2401   u8 proto = 9;
2402   u32 fib_index = 9000001;
2403   u32 thread_index = 3000000001;
2404   u32 session_index = 3000000221;
2405   clib_bihash_kv_16_8_t kv;
2406   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2407               thread_index, session_index);
2408   ip4_address_t l_addr2;
2409   ip4_address_t r_addr2;
2410   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2411   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2412   u16 l_port2 = 0;
2413   u16 r_port2 = 0;
2414   u8 proto2 = 0;
2415   u32 fib_index2 = 0;
2416   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2417                &r_port2);
2418   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2419   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2420   ASSERT (l_port == l_port2);
2421   ASSERT (r_port == r_port2);
2422   ASSERT (proto == proto2);
2423   ASSERT (fib_index == fib_index2);
2424   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2425   ASSERT (session_index == ed_value_get_session_index (&kv));
2426
2427   fib_index = 7001;
2428   proto = 5;
2429   nat_protocol_t proto3 = ~0;
2430   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2431   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2432   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2433   ASSERT (l_port == l_port2);
2434   ASSERT (proto == proto3);
2435   ASSERT (fib_index == fib_index2);
2436 }
2437
2438 static clib_error_t *
2439 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2440 {
2441   snat_main_t *sm = &snat_main;
2442   u32 fib_index;
2443
2444   if (sm->endpoint_dependent)
2445     {
2446       // TODO: consider removing all NAT interfaces
2447       if (!is_add)
2448         {
2449           fib_index = ip4_fib_index_from_table_id (table_id);
2450           if (fib_index != ~0)
2451             expire_per_vrf_sessions (fib_index);
2452         }
2453     }
2454   return 0;
2455 }
2456
2457 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2458
2459 void
2460 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2461 {
2462   vlib_node_t *node;
2463
2464   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2465   sm->ei_out2in_node_index = node->index;
2466   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2467   sm->ei_in2out_node_index = node->index;
2468   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2469   sm->ei_in2out_output_node_index = node->index;
2470
2471   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2472   sm->ed_out2in_node_index = node->index;
2473   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2474   sm->ed_in2out_node_index = node->index;
2475   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2476   sm->ed_in2out_output_node_index = node->index;
2477
2478   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2479   sm->error_node_index = node->index;
2480   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2481   sm->pre_in2out_node_index = node->index;
2482   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2483   sm->pre_out2in_node_index = node->index;
2484   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2485   sm->pre_in2out_node_index = node->index;
2486   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2487   sm->pre_out2in_node_index = node->index;
2488   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2489   sm->in2out_fast_node_index = node->index;
2490   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2491   sm->in2out_slowpath_node_index = node->index;
2492   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2493   sm->in2out_slowpath_output_node_index = node->index;
2494   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2495   sm->ed_in2out_slowpath_node_index = node->index;
2496   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2497   sm->out2in_fast_node_index = node->index;
2498   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2499   sm->ed_out2in_slowpath_node_index = node->index;
2500   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2501   sm->hairpinning_node_index = node->index;
2502   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2503   sm->hairpin_dst_node_index = node->index;
2504   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2505   sm->hairpin_src_node_index = node->index;
2506 }
2507
2508 #define nat_init_simple_counter(c, n, sn) \
2509 do                                        \
2510   {                                       \
2511     c.name = n;                           \
2512     c.stat_segment_name = sn;             \
2513     vlib_validate_simple_counter (&c, 0); \
2514     vlib_zero_simple_counter (&c, 0);     \
2515   } while (0);
2516
2517 extern vlib_node_registration_t nat44_hairpinning_node;
2518 extern vlib_node_registration_t snat_hairpin_dst_node;
2519 extern vlib_node_registration_t
2520   nat44_in2out_hairpinning_finish_ip4_lookup_node;
2521 extern vlib_node_registration_t
2522   nat44_in2out_hairpinning_finish_interface_output_node;
2523
2524 static clib_error_t *
2525 nat_init (vlib_main_t * vm)
2526 {
2527   snat_main_t *sm = &snat_main;
2528   vlib_thread_main_t *tm = vlib_get_thread_main ();
2529   vlib_thread_registration_t *tr;
2530   ip4_add_del_interface_address_callback_t cbi = { 0 };
2531   ip4_table_bind_callback_t cbt = { 0 };
2532   u32 i, num_threads = 0;
2533   uword *p, *bitmap = 0;
2534
2535   clib_memset (sm, 0, sizeof (*sm));
2536
2537   // required
2538   sm->vnet_main = vnet_get_main ();
2539   // convenience
2540   sm->ip4_main = &ip4_main;
2541   sm->api_main = vlibapi_get_main ();
2542   sm->ip4_lookup_main = &ip4_main.lookup_main;
2543
2544   // frame queue indices used for handoff
2545   sm->fq_out2in_index = ~0;
2546   sm->fq_in2out_index = ~0;
2547   sm->fq_in2out_output_index = ~0;
2548
2549   sm->log_level = SNAT_LOG_ERROR;
2550
2551   nat44_set_node_indexes (sm, vm);
2552   sm->log_class = vlib_log_register_class ("nat", 0);
2553   nat_ipfix_logging_init (vm);
2554
2555   nat_init_simple_counter (sm->total_users, "total-users",
2556                            "/nat44/total-users");
2557   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2558                            "/nat44/total-sessions");
2559   nat_init_simple_counter (sm->user_limit_reached, "user-limit-reached",
2560                            "/nat44/user-limit-reached");
2561
2562 #define _(x)                                            \
2563   sm->counters.fastpath.in2out.x.name = #x;             \
2564   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2565       "/nat44/in2out/fastpath/" #x;                     \
2566   sm->counters.slowpath.in2out.x.name = #x;             \
2567   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2568       "/nat44/in2out/slowpath/" #x;                     \
2569   sm->counters.fastpath.out2in.x.name = #x;             \
2570   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2571       "/nat44/out2in/fastpath/" #x;                     \
2572   sm->counters.slowpath.out2in.x.name = #x;             \
2573   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2574       "/nat44/out2in/slowpath/" #x;                     \
2575   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2576   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2577       "/nat44/ed/in2out/fastpath/" #x;                  \
2578   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2579   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2580       "/nat44/ed/in2out/slowpath/" #x;                  \
2581   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2582   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2583       "/nat44/ed/out2in/fastpath/" #x;                  \
2584   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2585   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2586       "/nat44/ed/out2in/slowpath/" #x;
2587   foreach_nat_counter;
2588 #undef _
2589   sm->counters.hairpinning.name = "hairpinning";
2590   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2591
2592   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2593   if (p)
2594     {
2595       tr = (vlib_thread_registration_t *) p[0];
2596       if (tr)
2597         {
2598           sm->num_workers = tr->count;
2599           sm->first_worker_index = tr->first_index;
2600         }
2601     }
2602   num_threads = tm->n_vlib_mains - 1;
2603   sm->port_per_thread = 0xffff - 1024;
2604   vec_validate (sm->per_thread_data, num_threads);
2605
2606   /* Use all available workers by default */
2607   if (sm->num_workers > 1)
2608     {
2609
2610       for (i = 0; i < sm->num_workers; i++)
2611         bitmap = clib_bitmap_set (bitmap, i, 1);
2612       snat_set_workers (bitmap);
2613       clib_bitmap_free (bitmap);
2614     }
2615   else
2616     sm->per_thread_data[0].snat_thread_index = 0;
2617
2618   /* callbacks to call when interface address changes. */
2619   cbi.function = snat_ip4_add_del_interface_address_cb;
2620   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2621   cbi.function = nat_ip4_add_del_addr_only_sm_cb;
2622   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2623
2624   /* callbacks to call when interface to table biding changes */
2625   cbt.function = snat_update_outside_fib;
2626   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2627
2628   sm->fib_src_low =
2629     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2630                          FIB_SOURCE_BH_SIMPLE);
2631   sm->fib_src_hi =
2632     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2633                          FIB_SOURCE_BH_SIMPLE);
2634
2635   /* used only by out2in-dpo feature */
2636   nat_dpo_module_init ();
2637
2638   nat_affinity_init (vm);
2639   nat_ha_init (vm, sm->num_workers, num_threads);
2640
2641   test_key_calc_split ();
2642
2643   sm->nat44_hairpinning_fq_index =
2644     vlib_frame_queue_main_init (nat44_hairpinning_node.index, 0);
2645   sm->snat_hairpin_dst_fq_index =
2646     vlib_frame_queue_main_init (snat_hairpin_dst_node.index, 0);
2647   sm->nat44_in2out_hairpinning_finish_ip4_lookup_node_fq_index =
2648     vlib_frame_queue_main_init (
2649       nat44_in2out_hairpinning_finish_ip4_lookup_node.index, 0);
2650   sm->nat44_in2out_hairpinning_finish_interface_output_node_fq_index =
2651     vlib_frame_queue_main_init (
2652       nat44_in2out_hairpinning_finish_interface_output_node.index, 0);
2653   return nat44_api_hookup (vm);
2654 }
2655
2656 VLIB_INIT_FUNCTION (nat_init);
2657
2658 static int
2659 nat44_ed_plugin_enable (nat44_config_t c)
2660 {
2661   snat_main_t *sm = &snat_main;
2662
2663   if (c.static_mapping_only && !c.connection_tracking)
2664     {
2665       nat_log_err ("unsupported combination of configuration");
2666       return 1;
2667     }
2668
2669   // nat44 feature configuration
2670   sm->endpoint_dependent = c.endpoint_dependent;
2671   sm->static_mapping_only = c.static_mapping_only;
2672   sm->static_mapping_connection_tracking = c.connection_tracking;
2673
2674   // EI only feature (could break ED)
2675   sm->out2in_dpo = c.out2in_dpo;
2676
2677   sm->forwarding_enabled = 0;
2678   sm->mss_clamping = 0;
2679   sm->pat = (!c.static_mapping_only ||
2680              (c.static_mapping_only && c.connection_tracking));
2681
2682   if (!c.users)
2683     c.users = 1024;
2684
2685   // EI only feature (could break ED)
2686   sm->max_users_per_thread = c.users;
2687   sm->user_buckets = nat_calc_bihash_buckets (c.users);
2688
2689   if (!c.sessions)
2690     c.sessions = 63 * 1024;
2691
2692   sm->max_translations_per_thread = c.sessions;
2693   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2694
2695   // ED only feature
2696   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2697
2698   // EI only feature (could break ED)
2699   sm->max_translations_per_user
2700     = c.user_sessions ? c.user_sessions : sm->max_translations_per_thread;
2701
2702   sm->inside_vrf_id = c.inside_vrf;
2703   sm->inside_fib_index =
2704     fib_table_find_or_create_and_lock
2705     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2706
2707   sm->outside_vrf_id = c.outside_vrf;
2708   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2709     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2710
2711   sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
2712   sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2713
2714   sm->in2out_node_index = sm->ed_in2out_node_index;
2715   sm->out2in_node_index = sm->ed_out2in_node_index;
2716
2717   sm->in2out_output_node_index = sm->ed_in2out_output_node_index;
2718
2719   if (sm->pat)
2720     {
2721       sm->icmp_match_in2out_cb = NULL;
2722       sm->icmp_match_out2in_cb = NULL;
2723     }
2724   else
2725     {
2726       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2727       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2728     }
2729
2730   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets,
2731                     sm->user_buckets);
2732
2733   nat_affinity_enable ();
2734
2735   nat_reset_timeouts (&sm->timeouts);
2736
2737   // TODO: function for reset counters
2738   vlib_zero_simple_counter (&sm->total_users, 0);
2739   vlib_zero_simple_counter (&sm->total_sessions, 0);
2740   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2741
2742   if (!sm->frame_queue_nelts)
2743     sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2744
2745   sm->enabled = 1;
2746   sm->rconfig = c;
2747
2748   return 0;
2749 }
2750
2751 int
2752 nat44_plugin_enable (nat44_config_t c)
2753 {
2754   fail_if_enabled ();
2755
2756   // c.static_mapping_only + c.connection_tracking
2757   //  - supported in NAT EI & NAT ED
2758   // c.out2in_dpo, c.static_mapping_only
2759   //  - supported in NAT EI
2760
2761   if (c.endpoint_dependent)
2762     {
2763       if (c.out2in_dpo || c.users || c.user_sessions)
2764         {
2765           nat_log_err ("unsupported combination of configuration");
2766           return 1;
2767         }
2768       return nat44_ed_plugin_enable (c);
2769     }
2770
2771   // separation:
2772   // for now just copy variables
2773
2774   nat44_ei_config_t ei_c = {
2775     .inside_vrf = c.inside_vrf,
2776     .outside_vrf = c.outside_vrf,
2777     .users = c.users,
2778     .sessions = c.sessions,
2779     .user_sessions = c.user_sessions,
2780     .out2in_dpo = c.out2in_dpo,
2781     .static_mapping_only = c.static_mapping_only,
2782     .connection_tracking = c.connection_tracking,
2783   };
2784
2785   return nat44_ei_plugin_enable (ei_c);
2786 }
2787
2788 void
2789 nat44_addresses_free (snat_address_t ** addresses)
2790 {
2791   snat_address_t *ap;
2792   /* *INDENT-OFF* */
2793   vec_foreach (ap, *addresses)
2794     {
2795     #define _(N, i, n, s) \
2796       vec_free (ap->busy_##n##_ports_per_thread);
2797       foreach_nat_protocol
2798     #undef _
2799     }
2800   /* *INDENT-ON* */
2801   vec_free (*addresses);
2802   *addresses = 0;
2803 }
2804
2805 static int
2806 nat44_ed_plugin_disable ()
2807 {
2808   snat_main_t *sm = &snat_main;
2809   snat_interface_t *i, *vec;
2810   int error = 0;
2811
2812   // first unregister all nodes from interfaces
2813   vec = vec_dup (sm->interfaces);
2814   /* *INDENT-OFF* */
2815   vec_foreach (i, vec)
2816     {
2817       if (nat_interface_is_inside(i))
2818         error = snat_interface_add_del (i->sw_if_index, 1, 1);
2819       if (nat_interface_is_outside(i))
2820         error = snat_interface_add_del (i->sw_if_index, 0, 1);
2821
2822       if (error)
2823         {
2824           nat_log_err ("error occurred while removing interface %u",
2825                        i->sw_if_index);
2826         }
2827     }
2828   /* *INDENT-ON* */
2829   vec_free (vec);
2830   sm->interfaces = 0;
2831
2832   vec = vec_dup (sm->output_feature_interfaces);
2833   /* *INDENT-OFF* */
2834   vec_foreach (i, vec)
2835     {
2836       if (nat_interface_is_inside(i))
2837         error = snat_interface_add_del_output_feature (i->sw_if_index, 1, 1);
2838       if (nat_interface_is_outside(i))
2839         error = snat_interface_add_del_output_feature (i->sw_if_index, 0, 1);
2840
2841       if (error)
2842         {
2843           nat_log_err ("error occurred while removing interface %u",
2844                        i->sw_if_index);
2845         }
2846     }
2847   /* *INDENT-ON* */
2848   vec_free (vec);
2849   sm->output_feature_interfaces = 0;
2850
2851   vec_free (sm->max_translations_per_fib);
2852
2853   nat44_ed_db_free ();
2854
2855   nat44_addresses_free (&sm->addresses);
2856   nat44_addresses_free (&sm->twice_nat_addresses);
2857
2858
2859   vec_free (sm->to_resolve);
2860   vec_free (sm->auto_add_sw_if_indices);
2861   vec_free (sm->auto_add_sw_if_indices_twice_nat);
2862
2863   sm->to_resolve = 0;
2864   sm->auto_add_sw_if_indices = 0;
2865   sm->auto_add_sw_if_indices_twice_nat = 0;
2866
2867   sm->forwarding_enabled = 0;
2868
2869   sm->enabled = 0;
2870   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2871
2872   return 0;
2873 }
2874
2875 int
2876 nat44_plugin_disable ()
2877 {
2878   snat_main_t *sm = &snat_main;
2879
2880   fail_if_disabled ();
2881
2882   if (sm->endpoint_dependent)
2883     return nat44_ed_plugin_disable ();
2884   return nat44_ei_plugin_disable ();
2885 }
2886
2887 void
2888 snat_free_outside_address_and_port (snat_address_t *addresses,
2889                                     u32 thread_index, ip4_address_t *addr,
2890                                     u16 port, nat_protocol_t protocol)
2891 {
2892   snat_address_t *a;
2893   u32 address_index;
2894   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2895
2896   for (address_index = 0; address_index < vec_len (addresses);
2897        address_index++)
2898     {
2899       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2900         break;
2901     }
2902
2903   ASSERT (address_index < vec_len (addresses));
2904
2905   a = addresses + address_index;
2906
2907   switch (protocol)
2908     {
2909 #define _(N, i, n, s) \
2910     case NAT_PROTOCOL_##N: \
2911       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2912       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2913       a->busy_##n##_ports--; \
2914       a->busy_##n##_ports_per_thread[thread_index]--; \
2915       break;
2916       foreach_nat_protocol
2917 #undef _
2918     default:
2919       nat_elog_info ("unknown protocol");
2920       return;
2921     }
2922 }
2923
2924 int
2925 nat_set_outside_address_and_port (snat_address_t *addresses, u32 thread_index,
2926                                   ip4_address_t addr, u16 port,
2927                                   nat_protocol_t protocol)
2928 {
2929   snat_address_t *a = 0;
2930   u32 address_index;
2931   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2932
2933   for (address_index = 0; address_index < vec_len (addresses);
2934        address_index++)
2935     {
2936       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2937         continue;
2938
2939       a = addresses + address_index;
2940       switch (protocol)
2941         {
2942 #define _(N, j, n, s) \
2943         case NAT_PROTOCOL_##N: \
2944           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2945             return VNET_API_ERROR_INSTANCE_IN_USE; \
2946           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2947           a->busy_##n##_ports_per_thread[thread_index]++; \
2948           a->busy_##n##_ports++; \
2949           return 0;
2950           foreach_nat_protocol
2951 #undef _
2952         default:
2953           nat_elog_info ("unknown protocol");
2954           return 1;
2955         }
2956     }
2957
2958   return VNET_API_ERROR_NO_SUCH_ENTRY;
2959 }
2960
2961 int
2962 snat_static_mapping_match (snat_main_t * sm,
2963                            ip4_address_t match_addr,
2964                            u16 match_port,
2965                            u32 match_fib_index,
2966                            nat_protocol_t match_protocol,
2967                            ip4_address_t * mapping_addr,
2968                            u16 * mapping_port,
2969                            u32 * mapping_fib_index,
2970                            u8 by_external,
2971                            u8 * is_addr_only,
2972                            twice_nat_type_t * twice_nat,
2973                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2974                            u8 * is_identity_nat, snat_static_mapping_t ** out)
2975 {
2976   clib_bihash_kv_8_8_t kv, value;
2977   clib_bihash_8_8_t *mapping_hash;
2978   snat_static_mapping_t *m;
2979   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2980   nat44_lb_addr_port_t *local;
2981   u8 backend_index;
2982
2983   if (!by_external)
2984     {
2985       mapping_hash = &sm->static_mapping_by_local;
2986       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2987                   match_protocol);
2988       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2989         {
2990           /* Try address only mapping */
2991           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2992           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2993             return 1;
2994         }
2995     }
2996   else
2997     {
2998       mapping_hash = &sm->static_mapping_by_external;
2999       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
3000       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3001         {
3002           /* Try address only mapping */
3003           init_nat_k (&kv, match_addr, 0, 0, 0);
3004           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3005             return 1;
3006         }
3007     }
3008
3009   m = pool_elt_at_index (sm->static_mappings, value.value);
3010
3011   if (by_external)
3012     {
3013       if (is_lb_static_mapping (m))
3014         {
3015           if (PREDICT_FALSE (lb != 0))
3016             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
3017           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
3018                                                           match_addr,
3019                                                           match_protocol,
3020                                                           match_port,
3021                                                           &backend_index))
3022             {
3023               local = pool_elt_at_index (m->locals, backend_index);
3024               *mapping_addr = local->addr;
3025               *mapping_port = local->port;
3026               *mapping_fib_index = local->fib_index;
3027               goto end;
3028             }
3029           // pick locals matching this worker
3030           if (PREDICT_FALSE (sm->num_workers > 1))
3031             {
3032               u32 thread_index = vlib_get_thread_index ();
3033               /* *INDENT-OFF* */
3034               pool_foreach_index (i, m->locals)
3035                {
3036                 local = pool_elt_at_index (m->locals, i);
3037
3038                 ip4_header_t ip = {
3039                   .src_address = local->addr,
3040                 };
3041
3042                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
3043                     thread_index)
3044                   {
3045                     vec_add1 (tmp, i);
3046                   }
3047               }
3048               /* *INDENT-ON* */
3049               ASSERT (vec_len (tmp) != 0);
3050             }
3051           else
3052             {
3053               /* *INDENT-OFF* */
3054               pool_foreach_index (i, m->locals)
3055                {
3056                 vec_add1 (tmp, i);
3057               }
3058               /* *INDENT-ON* */
3059             }
3060           hi = vec_len (tmp) - 1;
3061           local = pool_elt_at_index (m->locals, tmp[hi]);
3062           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
3063           while (lo < hi)
3064             {
3065               mid = ((hi - lo) >> 1) + lo;
3066               local = pool_elt_at_index (m->locals, tmp[mid]);
3067               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
3068             }
3069           local = pool_elt_at_index (m->locals, tmp[lo]);
3070           if (!(local->prefix >= rand))
3071             return 1;
3072           *mapping_addr = local->addr;
3073           *mapping_port = local->port;
3074           *mapping_fib_index = local->fib_index;
3075           if (m->affinity)
3076             {
3077               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
3078                                                 match_protocol, match_port,
3079                                                 tmp[lo], m->affinity,
3080                                                 m->affinity_per_service_list_head_index))
3081                 nat_elog_info ("create affinity record failed");
3082             }
3083           vec_free (tmp);
3084         }
3085       else
3086         {
3087           if (PREDICT_FALSE (lb != 0))
3088             *lb = NO_LB_NAT;
3089           *mapping_fib_index = m->fib_index;
3090           *mapping_addr = m->local_addr;
3091           /* Address only mapping doesn't change port */
3092           *mapping_port = is_addr_only_static_mapping (m) ? match_port
3093             : m->local_port;
3094         }
3095     }
3096   else
3097     {
3098       *mapping_addr = m->external_addr;
3099       /* Address only mapping doesn't change port */
3100       *mapping_port = is_addr_only_static_mapping (m) ? match_port
3101         : m->external_port;
3102       *mapping_fib_index = sm->outside_fib_index;
3103     }
3104
3105 end:
3106   if (PREDICT_FALSE (is_addr_only != 0))
3107     *is_addr_only = is_addr_only_static_mapping (m);
3108
3109   if (PREDICT_FALSE (twice_nat != 0))
3110     *twice_nat = m->twice_nat;
3111
3112   if (PREDICT_FALSE (is_identity_nat != 0))
3113     *is_identity_nat = is_identity_static_mapping (m);
3114
3115   if (out != 0)
3116     *out = m;
3117
3118   return 0;
3119 }
3120
3121 void
3122 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3123 {
3124   snat_main_t *sm = &snat_main;
3125   dpo_id_t dpo_v4 = DPO_INVALID;
3126   fib_prefix_t pfx = {
3127     .fp_proto = FIB_PROTOCOL_IP4,
3128     .fp_len = 32,
3129     .fp_addr.ip4.as_u32 = addr.as_u32,
3130   };
3131
3132   if (is_add)
3133     {
3134       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3135       fib_table_entry_special_dpo_add (0, &pfx, sm->fib_src_hi,
3136                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3137       dpo_reset (&dpo_v4);
3138     }
3139   else
3140     {
3141       fib_table_entry_special_remove (0, &pfx, sm->fib_src_hi);
3142     }
3143 }
3144
3145 static u32
3146 nat44_ed_get_worker_in2out_cb (ip4_header_t *ip, u32 rx_fib_index,
3147                                u8 is_output)
3148 {
3149   snat_main_t *sm = &snat_main;
3150   u32 next_worker_index = sm->first_worker_index;
3151   u32 hash;
3152
3153   clib_bihash_kv_16_8_t kv16, value16;
3154   snat_main_per_thread_data_t *tsm;
3155   udp_header_t *udp;
3156
3157   if (PREDICT_FALSE (is_output))
3158     {
3159       u32 fib_index = sm->outside_fib_index;
3160       nat_outside_fib_t *outside_fib;
3161       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3162       fib_prefix_t pfx = {
3163         .fp_proto = FIB_PROTOCOL_IP4,
3164         .fp_len = 32,
3165         .fp_addr = {
3166                     .ip4.as_u32 = ip->dst_address.as_u32,
3167                     }
3168         ,
3169       };
3170
3171       udp = ip4_next_header (ip);
3172
3173       switch (vec_len (sm->outside_fibs))
3174         {
3175         case 0:
3176           fib_index = sm->outside_fib_index;
3177           break;
3178         case 1:
3179           fib_index = sm->outside_fibs[0].fib_index;
3180           break;
3181         default:
3182             /* *INDENT-OFF* */
3183             vec_foreach (outside_fib, sm->outside_fibs)
3184               {
3185                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3186                 if (FIB_NODE_INDEX_INVALID != fei)
3187                   {
3188                     if (fib_entry_get_resolving_interface (fei) != ~0)
3189                       {
3190                         fib_index = outside_fib->fib_index;
3191                         break;
3192                       }
3193                   }
3194               }
3195             /* *INDENT-ON* */
3196           break;
3197         }
3198
3199       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3200                  udp->dst_port, fib_index, ip->protocol);
3201
3202       if (PREDICT_TRUE (
3203             !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3204         {
3205           tsm =
3206             vec_elt_at_index (sm->per_thread_data,
3207                               ed_value_get_thread_index (&value16));
3208           next_worker_index += tsm->thread_index;
3209
3210           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3211                                   next_worker_index, fib_index,
3212                                   clib_net_to_host_u32 (ip->
3213                                                         src_address.as_u32),
3214                                   clib_net_to_host_u32 (ip->
3215                                                         dst_address.as_u32));
3216
3217           return next_worker_index;
3218         }
3219     }
3220
3221   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3222     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3223
3224   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3225     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3226   else
3227     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3228
3229   if (PREDICT_TRUE (!is_output))
3230     {
3231       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3232                               next_worker_index, rx_fib_index,
3233                               clib_net_to_host_u32 (ip->src_address.as_u32),
3234                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3235     }
3236   else
3237     {
3238       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3239                               next_worker_index, rx_fib_index,
3240                               clib_net_to_host_u32 (ip->src_address.as_u32),
3241                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3242     }
3243
3244   return next_worker_index;
3245 }
3246
3247 static u32
3248 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3249                                u32 rx_fib_index, u8 is_output)
3250 {
3251   snat_main_t *sm = &snat_main;
3252   clib_bihash_kv_8_8_t kv, value;
3253   clib_bihash_kv_16_8_t kv16, value16;
3254   snat_main_per_thread_data_t *tsm;
3255
3256   u32 proto, next_worker_index = 0;
3257   udp_header_t *udp;
3258   u16 port;
3259   snat_static_mapping_t *m;
3260   u32 hash;
3261
3262   proto = ip_proto_to_nat_proto (ip->protocol);
3263
3264   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3265     {
3266       udp = ip4_next_header (ip);
3267
3268       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3269                  udp->src_port, rx_fib_index, ip->protocol);
3270
3271       if (PREDICT_TRUE (
3272             !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3273         {
3274           tsm =
3275             vec_elt_at_index (sm->per_thread_data,
3276                               ed_value_get_thread_index (&value16));
3277           vnet_buffer2 (b)->nat.cached_session_index =
3278             ed_value_get_session_index (&value16);
3279           next_worker_index = sm->first_worker_index + tsm->thread_index;
3280           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3281                                   next_worker_index, rx_fib_index,
3282                                   clib_net_to_host_u32 (ip->
3283                                                         src_address.as_u32),
3284                                   clib_net_to_host_u32 (ip->
3285                                                         dst_address.as_u32));
3286           return next_worker_index;
3287         }
3288     }
3289   else if (proto == NAT_PROTOCOL_ICMP)
3290     {
3291       ip4_address_t lookup_saddr, lookup_daddr;
3292       u16 lookup_sport, lookup_dport;
3293       u8 lookup_protocol;
3294       if (!nat_get_icmp_session_lookup_values (
3295             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3296             &lookup_protocol))
3297         {
3298           init_ed_k (&kv16, lookup_saddr, lookup_sport, lookup_daddr,
3299                      lookup_dport, rx_fib_index, lookup_protocol);
3300           if (PREDICT_TRUE (
3301                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3302             {
3303               tsm =
3304                 vec_elt_at_index (sm->per_thread_data,
3305                                   ed_value_get_thread_index (&value16));
3306               next_worker_index = sm->first_worker_index + tsm->thread_index;
3307               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3308                                       next_worker_index, rx_fib_index,
3309                                       clib_net_to_host_u32 (ip->
3310                                                             src_address.as_u32),
3311                                       clib_net_to_host_u32 (ip->
3312                                                             dst_address.as_u32));
3313               return next_worker_index;
3314             }
3315         }
3316     }
3317
3318   /* first try static mappings without port */
3319   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3320     {
3321       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3322       if (!clib_bihash_search_8_8
3323           (&sm->static_mapping_by_external, &kv, &value))
3324         {
3325           m = pool_elt_at_index (sm->static_mappings, value.value);
3326           next_worker_index = m->workers[0];
3327           goto done;
3328         }
3329     }
3330
3331   /* unknown protocol */
3332   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3333     {
3334       /* use current thread */
3335       next_worker_index = vlib_get_thread_index ();
3336       goto done;
3337     }
3338
3339   udp = ip4_next_header (ip);
3340   port = udp->dst_port;
3341
3342   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3343     {
3344       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3345       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3346       if (!icmp_type_is_error_message
3347           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3348         port = vnet_buffer (b)->ip.reass.l4_src_port;
3349       else
3350         {
3351           /* if error message, then it's not fragmented and we can access it */
3352           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3353           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3354           void *l4_header = ip4_next_header (inner_ip);
3355           switch (proto)
3356             {
3357             case NAT_PROTOCOL_ICMP:
3358               icmp = (icmp46_header_t *) l4_header;
3359               echo = (icmp_echo_header_t *) (icmp + 1);
3360               port = echo->identifier;
3361               break;
3362             case NAT_PROTOCOL_UDP:
3363             case NAT_PROTOCOL_TCP:
3364               port = ((tcp_udp_header_t *) l4_header)->src_port;
3365               break;
3366             default:
3367               next_worker_index = vlib_get_thread_index ();
3368               goto done;
3369             }
3370         }
3371     }
3372
3373   /* try static mappings with port */
3374   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3375     {
3376       init_nat_k (&kv, ip->dst_address, port, 0, proto);
3377       if (!clib_bihash_search_8_8
3378           (&sm->static_mapping_by_external, &kv, &value))
3379         {
3380           m = pool_elt_at_index (sm->static_mappings, value.value);
3381           if (!is_lb_static_mapping (m))
3382             {
3383               next_worker_index = m->workers[0];
3384               goto done;
3385             }
3386
3387           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3388             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3389
3390           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3391             next_worker_index =
3392               m->workers[hash & (_vec_len (m->workers) - 1)];
3393           else
3394             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3395           goto done;
3396         }
3397     }
3398
3399   /* worker by outside port */
3400   next_worker_index = sm->first_worker_index;
3401   next_worker_index +=
3402     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3403
3404 done:
3405   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3406                           clib_net_to_host_u32 (ip->src_address.as_u32),
3407                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3408   return next_worker_index;
3409 }
3410
3411 u32
3412 nat_calc_bihash_buckets (u32 n_elts)
3413 {
3414   n_elts = n_elts / 2.5;
3415   u64 lower_pow2 = 1;
3416   while (lower_pow2 * 2 < n_elts)
3417     {
3418       lower_pow2 = 2 * lower_pow2;
3419     }
3420   u64 upper_pow2 = 2 * lower_pow2;
3421   if ((upper_pow2 - n_elts) < (n_elts - lower_pow2))
3422     {
3423       if (upper_pow2 <= UINT32_MAX)
3424         {
3425           return upper_pow2;
3426         }
3427     }
3428   return lower_pow2;
3429 }
3430
3431 u32
3432 nat44_get_max_session_limit ()
3433 {
3434   snat_main_t *sm = &snat_main;
3435   u32 max_limit = 0, len = 0;
3436
3437   for (; len < vec_len (sm->max_translations_per_fib); len++)
3438     {
3439       if (max_limit < sm->max_translations_per_fib[len])
3440         max_limit = sm->max_translations_per_fib[len];
3441     }
3442   return max_limit;
3443 }
3444
3445 int
3446 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3447 {
3448   snat_main_t *sm = &snat_main;
3449   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3450   u32 len = vec_len (sm->max_translations_per_fib);
3451
3452   if (len <= fib_index)
3453     {
3454       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3455
3456       for (; len < vec_len (sm->max_translations_per_fib); len++)
3457         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3458     }
3459
3460   sm->max_translations_per_fib[fib_index] = session_limit;
3461   return 0;
3462 }
3463
3464 int
3465 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3466 {
3467   snat_main_t *sm = &snat_main;
3468
3469   if (nat44_set_session_limit (session_limit, vrf_id))
3470     return 1;
3471   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3472
3473   sm->translation_buckets =
3474     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3475
3476   nat44_sessions_clear ();
3477   return 0;
3478 }
3479
3480 static void
3481 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
3482                          u32 translation_buckets, u32 user_buckets)
3483 {
3484   dlist_elt_t *head;
3485
3486   pool_alloc (tsm->sessions, translations);
3487   pool_alloc (tsm->lru_pool, translations);
3488
3489   pool_get (tsm->lru_pool, head);
3490   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3491   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3492
3493   pool_get (tsm->lru_pool, head);
3494   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3495   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3496
3497   pool_get (tsm->lru_pool, head);
3498   tsm->udp_lru_head_index = head - tsm->lru_pool;
3499   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3500
3501   pool_get (tsm->lru_pool, head);
3502   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3503   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3504
3505   pool_get (tsm->lru_pool, head);
3506   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3507   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3508
3509   // TODO: ED nat is not using these
3510   // before removal large refactor required
3511   pool_alloc (tsm->list_pool, translations);
3512   clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets, 0);
3513   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3514 }
3515
3516 static void
3517 reinit_ed_flow_hash ()
3518 {
3519   snat_main_t *sm = &snat_main;
3520   // we expect 2 flows per session, so multiply translation_buckets by 2
3521   clib_bihash_init_16_8 (
3522     &sm->flow_hash, "ed-flow-hash",
3523     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3524   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3525 }
3526
3527 static void
3528 nat44_ed_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
3529 {
3530   snat_main_t *sm = &snat_main;
3531   snat_main_per_thread_data_t *tsm;
3532   u32 static_mapping_buckets = 1024;
3533   u32 static_mapping_memory_size = 64 << 20;
3534
3535   reinit_ed_flow_hash ();
3536
3537   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3538                         "static_mapping_by_local", static_mapping_buckets,
3539                         static_mapping_memory_size);
3540   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3541                                      format_static_mapping_kvp);
3542
3543   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3544                         "static_mapping_by_external", static_mapping_buckets,
3545                         static_mapping_memory_size);
3546   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3547                                      format_static_mapping_kvp);
3548
3549   if (sm->pat)
3550     {
3551       vec_foreach (tsm, sm->per_thread_data)
3552         {
3553           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3554                                    sm->translation_buckets, sm->user_buckets);
3555         }
3556     }
3557 }
3558
3559 static void
3560 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3561 {
3562   pool_free (tsm->sessions);
3563   pool_free (tsm->lru_pool);
3564
3565   vec_free (tsm->per_vrf_sessions_vec);
3566
3567   // TODO: resolve static mappings (put only to !ED)
3568   pool_free (tsm->list_pool);
3569   pool_free (tsm->users);
3570   clib_bihash_free_8_8 (&tsm->user_hash);
3571 }
3572
3573 static void
3574 nat44_ed_db_free ()
3575 {
3576   snat_main_t *sm = &snat_main;
3577   snat_main_per_thread_data_t *tsm;
3578
3579   pool_free (sm->static_mappings);
3580   clib_bihash_free_16_8 (&sm->flow_hash);
3581   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
3582   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
3583
3584   if (sm->pat)
3585     {
3586       vec_foreach (tsm, sm->per_thread_data)
3587         {
3588           nat44_ed_worker_db_free (tsm);
3589         }
3590     }
3591 }
3592
3593 void
3594 nat44_ed_sessions_clear ()
3595 {
3596   snat_main_t *sm = &snat_main;
3597   snat_main_per_thread_data_t *tsm;
3598
3599   reinit_ed_flow_hash ();
3600
3601   if (sm->pat)
3602     {
3603       vec_foreach (tsm, sm->per_thread_data)
3604         {
3605
3606           nat44_ed_worker_db_free (tsm);
3607           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3608                                    sm->translation_buckets, sm->user_buckets);
3609         }
3610     }
3611
3612   // TODO: function for reset counters
3613   vlib_zero_simple_counter (&sm->total_users, 0);
3614   vlib_zero_simple_counter (&sm->total_sessions, 0);
3615   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
3616 }
3617
3618 void
3619 nat44_sessions_clear ()
3620 {
3621   snat_main_t *sm = &snat_main;
3622
3623   if (sm->endpoint_dependent)
3624     nat44_ed_sessions_clear ();
3625   else
3626     nat44_ei_sessions_clear ();
3627 }
3628
3629 static void
3630 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3631                                  uword opaque,
3632                                  u32 sw_if_index,
3633                                  ip4_address_t * address,
3634                                  u32 address_length,
3635                                  u32 if_address_index, u32 is_delete)
3636 {
3637   snat_main_t *sm = &snat_main;
3638   snat_static_map_resolve_t *rp;
3639   snat_static_mapping_t *m;
3640   clib_bihash_kv_8_8_t kv, value;
3641   int i, rv;
3642   ip4_address_t l_addr;
3643
3644   if (!sm->enabled)
3645     return;
3646
3647   for (i = 0; i < vec_len (sm->to_resolve); i++)
3648     {
3649       rp = sm->to_resolve + i;
3650       if (rp->addr_only == 0)
3651         continue;
3652       if (rp->sw_if_index == sw_if_index)
3653         goto match;
3654     }
3655
3656   return;
3657
3658 match:
3659   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
3660               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
3661   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3662     m = 0;
3663   else
3664     m = pool_elt_at_index (sm->static_mappings, value.value);
3665
3666   if (!is_delete)
3667     {
3668       /* Don't trip over lease renewal, static config */
3669       if (m)
3670         return;
3671     }
3672   else
3673     {
3674       if (!m)
3675         return;
3676     }
3677
3678   /* Indetity mapping? */
3679   if (rp->l_addr.as_u32 == 0)
3680     l_addr.as_u32 = address[0].as_u32;
3681   else
3682     l_addr.as_u32 = rp->l_addr.as_u32;
3683   /* Add the static mapping */
3684   rv = snat_add_static_mapping (l_addr,
3685                                 address[0],
3686                                 rp->l_port,
3687                                 rp->e_port,
3688                                 rp->vrf_id,
3689                                 rp->addr_only, ~0 /* sw_if_index */ ,
3690                                 rp->proto, !is_delete, rp->twice_nat,
3691                                 rp->out2in_only, rp->tag, rp->identity_nat,
3692                                 rp->pool_addr, rp->exact);
3693   if (rv)
3694     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
3695 }
3696
3697 static void
3698 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3699                                        uword opaque,
3700                                        u32 sw_if_index,
3701                                        ip4_address_t * address,
3702                                        u32 address_length,
3703                                        u32 if_address_index, u32 is_delete)
3704 {
3705   snat_main_t *sm = &snat_main;
3706   snat_static_map_resolve_t *rp;
3707   ip4_address_t l_addr;
3708   int i, j;
3709   int rv;
3710   u8 twice_nat = 0;
3711   snat_address_t *addresses = sm->addresses;
3712
3713   if (!sm->enabled)
3714     return;
3715
3716   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3717     {
3718       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3719         goto match;
3720     }
3721
3722   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3723     {
3724       twice_nat = 1;
3725       addresses = sm->twice_nat_addresses;
3726       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3727         goto match;
3728     }
3729
3730   return;
3731
3732 match:
3733   if (!is_delete)
3734     {
3735       /* Don't trip over lease renewal, static config */
3736       for (j = 0; j < vec_len (addresses); j++)
3737         if (addresses[j].addr.as_u32 == address->as_u32)
3738           return;
3739
3740       (void) snat_add_address (sm, address, ~0, twice_nat);
3741       /* Scan static map resolution vector */
3742       for (j = 0; j < vec_len (sm->to_resolve); j++)
3743         {
3744           rp = sm->to_resolve + j;
3745           if (rp->addr_only)
3746             continue;
3747           /* On this interface? */
3748           if (rp->sw_if_index == sw_if_index)
3749             {
3750               /* Indetity mapping? */
3751               if (rp->l_addr.as_u32 == 0)
3752                 l_addr.as_u32 = address[0].as_u32;
3753               else
3754                 l_addr.as_u32 = rp->l_addr.as_u32;
3755               /* Add the static mapping */
3756               rv = snat_add_static_mapping (
3757                 l_addr, address[0], rp->l_port, rp->e_port, rp->vrf_id,
3758                 rp->addr_only, ~0 /* sw_if_index */, rp->proto, 1,
3759                 rp->twice_nat, rp->out2in_only, rp->tag, rp->identity_nat,
3760                 rp->pool_addr, rp->exact);
3761               if (rv)
3762                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
3763                                     "i4", rv);
3764             }
3765         }
3766       return;
3767     }
3768   else
3769     {
3770       (void) snat_del_address (sm, address[0], 1, twice_nat);
3771       return;
3772     }
3773 }
3774
3775 int
3776 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3777                             u8 twice_nat)
3778 {
3779   ip4_main_t *ip4_main = sm->ip4_main;
3780   ip4_address_t *first_int_addr;
3781   snat_static_map_resolve_t *rp;
3782   u32 *indices_to_delete = 0;
3783   int i, j;
3784   u32 *auto_add_sw_if_indices =
3785     twice_nat ? sm->
3786     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3787
3788   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3789     );
3790
3791   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3792     {
3793       if (auto_add_sw_if_indices[i] == sw_if_index)
3794         {
3795           if (is_del)
3796             {
3797               /* if have address remove it */
3798               if (first_int_addr)
3799                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3800               else
3801                 {
3802                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3803                     {
3804                       rp = sm->to_resolve + j;
3805                       if (rp->sw_if_index == sw_if_index)
3806                         vec_add1 (indices_to_delete, j);
3807                     }
3808                   if (vec_len (indices_to_delete))
3809                     {
3810                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3811                         vec_del1 (sm->to_resolve, j);
3812                       vec_free (indices_to_delete);
3813                     }
3814                 }
3815               if (twice_nat)
3816                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3817               else
3818                 vec_del1 (sm->auto_add_sw_if_indices, i);
3819             }
3820           else
3821             return VNET_API_ERROR_VALUE_EXIST;
3822
3823           return 0;
3824         }
3825     }
3826
3827   if (is_del)
3828     return VNET_API_ERROR_NO_SUCH_ENTRY;
3829
3830   /* add to the auto-address list */
3831   if (twice_nat)
3832     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3833   else
3834     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3835
3836   /* If the address is already bound - or static - add it now */
3837   if (first_int_addr)
3838     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3839
3840   return 0;
3841 }
3842
3843 int
3844 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3845                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3846                       u32 vrf_id, int is_in)
3847 {
3848   ip4_header_t ip;
3849   clib_bihash_kv_16_8_t kv, value;
3850   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3851   snat_session_t *s;
3852   snat_main_per_thread_data_t *tsm;
3853
3854   if (!sm->endpoint_dependent)
3855     return VNET_API_ERROR_FEATURE_DISABLED;
3856
3857   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3858   if (sm->num_workers > 1)
3859     tsm =
3860       vec_elt_at_index (sm->per_thread_data,
3861                         sm->worker_in2out_cb (&ip, fib_index, 0));
3862   else
3863     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3864
3865   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
3866   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3867     {
3868       return VNET_API_ERROR_NO_SUCH_ENTRY;
3869     }
3870
3871   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3872     return VNET_API_ERROR_UNSPECIFIED;
3873   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3874   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3875   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3876   return 0;
3877 }
3878
3879 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3880                                  vlib_node_runtime_t * node,
3881                                  vlib_frame_t * frame)
3882 {
3883   return 0;
3884 }
3885
3886 /* *INDENT-OFF* */
3887 VLIB_REGISTER_NODE (nat_default_node) = {
3888   .name = "nat-default",
3889   .vector_size = sizeof (u32),
3890   .format_trace = 0,
3891   .type = VLIB_NODE_TYPE_INTERNAL,
3892   .n_errors = 0,
3893   .n_next_nodes = NAT_N_NEXT,
3894   .next_nodes = {
3895     [NAT_NEXT_DROP] = "error-drop",
3896     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3897     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3898     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3899     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3900     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3901     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3902     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3903     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3904     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3905   },
3906 };
3907 /* *INDENT-ON* */
3908
3909 void
3910 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3911 {
3912   f->l3_csum_delta = 0;
3913   f->l4_csum_delta = 0;
3914   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3915       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3916     {
3917       f->l3_csum_delta =
3918         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3919       f->l3_csum_delta =
3920         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3921     }
3922   else
3923     {
3924       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3925     }
3926   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3927       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3928     {
3929       f->l3_csum_delta =
3930         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3931       f->l3_csum_delta =
3932         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3933     }
3934   else
3935     {
3936       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3937     }
3938   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3939     {
3940       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3941       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3942     }
3943   else
3944     {
3945       f->rewrite.sport = f->match.sport;
3946     }
3947   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3948     {
3949       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3950       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3951     }
3952   else
3953     {
3954       f->rewrite.dport = f->match.dport;
3955     }
3956   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3957       f->rewrite.icmp_id != f->match.icmp_id)
3958     {
3959       f->l4_csum_delta =
3960         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3961       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.icmp_id);
3962     }
3963   else
3964     {
3965       f->rewrite.icmp_id = f->match.icmp_id;
3966     }
3967   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3968     {
3969     }
3970   else
3971     {
3972       f->rewrite.fib_index = f->match.fib_index;
3973     }
3974 }
3975
3976 static_always_inline int nat_6t_flow_icmp_translate (snat_main_t *sm,
3977                                                      vlib_buffer_t *b,
3978                                                      ip4_header_t *ip,
3979                                                      nat_6t_flow_t *f);
3980
3981 static_always_inline void
3982 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3983                            nat_6t_flow_t *f, nat_protocol_t proto,
3984                            int is_icmp_inner_ip4)
3985 {
3986   udp_header_t *udp = ip4_next_header (ip);
3987   tcp_header_t *tcp = (tcp_header_t *) udp;
3988
3989   if ((NAT_PROTOCOL_TCP == proto || NAT_PROTOCOL_UDP == proto) &&
3990       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3991     {
3992       if (!is_icmp_inner_ip4)
3993         { // regular case
3994           ip->src_address = f->rewrite.saddr;
3995           ip->dst_address = f->rewrite.daddr;
3996           udp->src_port = f->rewrite.sport;
3997           udp->dst_port = f->rewrite.dport;
3998         }
3999       else
4000         { // icmp inner ip4 - reversed saddr/daddr
4001           ip->src_address = f->rewrite.daddr;
4002           ip->dst_address = f->rewrite.saddr;
4003           udp->src_port = f->rewrite.dport;
4004           udp->dst_port = f->rewrite.sport;
4005         }
4006
4007       if (NAT_PROTOCOL_TCP == proto)
4008         {
4009           ip_csum_t tcp_sum = tcp->checksum;
4010           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
4011           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
4012           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
4013           tcp->checksum = ip_csum_fold (tcp_sum);
4014         }
4015       else if (proto == NAT_PROTOCOL_UDP && udp->checksum)
4016         {
4017           ip_csum_t udp_sum = udp->checksum;
4018           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
4019           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
4020           udp->checksum = ip_csum_fold (udp_sum);
4021         }
4022     }
4023   else
4024     {
4025       if (!is_icmp_inner_ip4)
4026         { // regular case
4027           ip->src_address = f->rewrite.saddr;
4028           ip->dst_address = f->rewrite.daddr;
4029         }
4030       else
4031         { // icmp inner ip4 - reversed saddr/daddr
4032           ip->src_address = f->rewrite.daddr;
4033           ip->dst_address = f->rewrite.saddr;
4034         }
4035     }
4036
4037   ip_csum_t ip_sum = ip->checksum;
4038   ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
4039   ip->checksum = ip_csum_fold (ip_sum);
4040   ASSERT (ip->checksum == ip4_header_checksum (ip));
4041 }
4042
4043 static_always_inline int
4044 nat_6t_flow_icmp_translate (snat_main_t *sm, vlib_buffer_t *b,
4045                             ip4_header_t *ip, nat_6t_flow_t *f)
4046 {
4047   if (IP_PROTOCOL_ICMP != ip->protocol)
4048     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4049
4050   icmp46_header_t *icmp = ip4_next_header (ip);
4051   icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
4052
4053   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
4054     {
4055       if (icmp->checksum == 0)
4056         icmp->checksum = 0xffff;
4057
4058       if (!icmp_type_is_error_message (icmp->type))
4059         {
4060           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
4061               (f->rewrite.icmp_id != echo->identifier))
4062             {
4063               ip_csum_t sum = icmp->checksum;
4064               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
4065                                     icmp_echo_header_t,
4066                                     identifier /* changed member */);
4067               echo->identifier = f->rewrite.icmp_id;
4068               icmp->checksum = ip_csum_fold (sum);
4069             }
4070         }
4071       else
4072         {
4073           // errors are not fragmented
4074           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
4075
4076           if (!ip4_header_checksum_is_valid (inner_ip))
4077             {
4078               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4079             }
4080
4081           nat_protocol_t inner_proto =
4082             ip_proto_to_nat_proto (inner_ip->protocol);
4083
4084           ip_csum_t icmp_sum = icmp->checksum;
4085
4086           switch (inner_proto)
4087             {
4088             case NAT_PROTOCOL_UDP:
4089             case NAT_PROTOCOL_TCP:
4090               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
4091                                          1 /* is_icmp_inner_ip4 */);
4092               icmp_sum = ip_csum_sub_even (icmp_sum, f->l3_csum_delta);
4093               icmp->checksum = ip_csum_fold (icmp_sum);
4094               break;
4095             case NAT_PROTOCOL_ICMP:
4096               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
4097                 {
4098                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
4099                   icmp_echo_header_t *inner_echo =
4100                     (icmp_echo_header_t *) (inner_icmp + 1);
4101                   if (f->rewrite.icmp_id != inner_echo->identifier)
4102                     {
4103                       ip_csum_t sum = icmp->checksum;
4104                       sum = ip_csum_update (
4105                         sum, inner_echo->identifier, f->rewrite.icmp_id,
4106                         icmp_echo_header_t, identifier /* changed member */);
4107                       icmp->checksum = ip_csum_fold (sum);
4108                       ip_csum_t inner_sum = inner_icmp->checksum;
4109                       inner_sum = ip_csum_update (
4110                         sum, inner_echo->identifier, f->rewrite.icmp_id,
4111                         icmp_echo_header_t, identifier /* changed member */);
4112                       inner_icmp->checksum = ip_csum_fold (inner_sum);
4113                       inner_echo->identifier = f->rewrite.icmp_id;
4114                     }
4115                 }
4116               break;
4117             default:
4118               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
4119               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
4120             }
4121         }
4122     }
4123   return NAT_ED_TRNSL_ERR_SUCCESS;
4124 }
4125
4126 nat_translation_error_e
4127 nat_6t_flow_buf_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
4128                            nat_6t_flow_t *f, nat_protocol_t proto,
4129                            int is_output_feature)
4130 {
4131   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
4132     {
4133       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
4134     }
4135
4136   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */);
4137
4138   if (NAT_PROTOCOL_ICMP == proto)
4139     {
4140       return nat_6t_flow_icmp_translate (sm, b, ip, f);
4141     }
4142
4143   return NAT_ED_TRNSL_ERR_SUCCESS;
4144 }
4145
4146 u8 *
4147 format_nat_6t (u8 *s, va_list *args)
4148 {
4149   nat_6t_t *t = va_arg (*args, nat_6t_t *);
4150
4151   s = format (s, "saddr %U sport %u daddr %U dport %u proto %U fib_idx %u",
4152               format_ip4_address, t->saddr.as_u8,
4153               clib_net_to_host_u16 (t->sport), format_ip4_address,
4154               t->daddr.as_u8, clib_net_to_host_u16 (t->dport),
4155               format_ip_protocol, t->proto, t->fib_index);
4156   return s;
4157 }
4158
4159 u8 *
4160 format_nat_ed_translation_error (u8 *s, va_list *args)
4161 {
4162   nat_translation_error_e e = va_arg (*args, nat_translation_error_e);
4163
4164   switch (e)
4165     {
4166     case NAT_ED_TRNSL_ERR_SUCCESS:
4167       s = format (s, "success");
4168       break;
4169     case NAT_ED_TRNSL_ERR_TRANSLATION_FAILED:
4170       s = format (s, "translation-failed");
4171       break;
4172     case NAT_ED_TRNSL_ERR_FLOW_MISMATCH:
4173       s = format (s, "flow-mismatch");
4174       break;
4175     }
4176   return s;
4177 }
4178
4179 u8 *
4180 format_nat_6t_flow (u8 *s, va_list *args)
4181 {
4182   nat_6t_flow_t *f = va_arg (*args, nat_6t_flow_t *);
4183
4184   s = format (s, "match: %U ", format_nat_6t, &f->match);
4185   int r = 0;
4186   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE)
4187     {
4188       s = format (s, "rewrite: saddr %U ", format_ip4_address,
4189                   f->rewrite.saddr.as_u8);
4190       r = 1;
4191     }
4192   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE)
4193     {
4194       if (!r)
4195         {
4196           s = format (s, "rewrite: ");
4197           r = 1;
4198         }
4199       s = format (s, "sport %u ", clib_net_to_host_u16 (f->rewrite.sport));
4200     }
4201   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE)
4202     {
4203       if (!r)
4204         {
4205           s = format (s, "rewrite: ");
4206           r = 1;
4207         }
4208       s = format (s, "daddr %U ", format_ip4_address, f->rewrite.daddr.as_u8);
4209     }
4210   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE)
4211     {
4212       if (!r)
4213         {
4214           s = format (s, "rewrite: ");
4215           r = 1;
4216         }
4217       s = format (s, "dport %u ", clib_net_to_host_u16 (f->rewrite.dport));
4218     }
4219   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
4220     {
4221       if (!r)
4222         {
4223           s = format (s, "rewrite: ");
4224           r = 1;
4225         }
4226       s = format (s, "icmp-id %u ", clib_net_to_host_u16 (f->rewrite.icmp_id));
4227     }
4228   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
4229     {
4230       if (!r)
4231         {
4232           s = format (s, "rewrite: ");
4233           r = 1;
4234         }
4235       s = format (s, "txfib %u ", f->rewrite.fib_index);
4236     }
4237   return s;
4238 }
4239
4240 /*
4241  * fd.io coding-style-patch-verification: ON
4242  *
4243  * Local Variables:
4244  * eval: (c-set-style "gnu")
4245  * End:
4246  */