src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <vnet/fib/fib_table.h>
  35 #include <vnet/fib/ip4_fib.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 /* *INDENT-OFF* */
  42
  43 /* Hook up input features */
  44 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  45   .arc_name = "ip4-unicast",
  46   .node_name = "nat44-in2out",
  47   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  48 };
  49 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  50   .arc_name = "ip4-unicast",
  51   .node_name = "nat44-out2in",
  52   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  53                                "ip4-dhcp-client-detect"),
  54 };
  55 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  56   .arc_name = "ip4-unicast",
  57   .node_name = "nat44-classify",
  58   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  59 };
  60 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  61   .arc_name = "ip4-unicast",
  62   .node_name = "nat44-det-in2out",
  63   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  64 };
  65 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  66   .arc_name = "ip4-unicast",
  67   .node_name = "nat44-det-out2in",
  68   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  69                                "ip4-dhcp-client-detect"),
  70 };
  71 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  72   .arc_name = "ip4-unicast",
  73   .node_name = "nat44-det-classify",
  74   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  75 };
  76 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  77   .arc_name = "ip4-unicast",
  78   .node_name = "nat44-ed-in2out",
  79   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-ed-out2in",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  85                                "ip4-dhcp-client-detect"),
  86 };
  87 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  88   .arc_name = "ip4-unicast",
  89   .node_name = "nat44-ed-classify",
  90   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  91 };
  92 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-in2out-worker-handoff",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  96 };
  97 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-out2in-worker-handoff",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 101                                "ip4-dhcp-client-detect"),
 102 };
 103 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-handoff-classify",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 107 };
 108 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-in2out-fast",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 112 };
 113 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-out2in-fast",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 117                                "ip4-dhcp-client-detect"),
 118 };
 119 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-hairpin-dst",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 123 };
 124 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-ed-hairpin-dst",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 128 };
 129
 130 /* Hook up output features */
 131 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 132   .arc_name = "ip4-output",
 133   .node_name = "nat44-in2out-output",
 134   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 137   .arc_name = "ip4-output",
 138   .node_name = "nat44-in2out-output-worker-handoff",
 139   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 140 };
 141 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 142   .arc_name = "ip4-output",
 143   .node_name = "nat44-hairpin-src",
 144   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 145 };
 146 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-ed-in2out-output",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 150 };
 151 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-ed-hairpin-src",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 155 };
 156
 157 /* Hook up ip4-local features */
 158 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 159 {
 160   .arc_name = "ip4-local",
 161   .node_name = "nat44-hairpinning",
 162   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 163 };
 164 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 165 {
 166   .arc_name = "ip4-local",
 167   .node_name = "nat44-ed-hairpinning",
 168   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 169 };
 170
 171
 172 VLIB_PLUGIN_REGISTER () = {
 173     .version = VPP_BUILD_VER,
 174     .description = "Network Address Translation",
 175 };
 176 /* *INDENT-ON* */
 177
 178 void
 179 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 180                        u8 is_ha)
 181 {
 182   snat_session_key_t key;
 183   clib_bihash_kv_8_8_t kv;
 184   nat_ed_ses_key_t ed_key;
 185   clib_bihash_kv_16_8_t ed_kv;
 186   snat_main_per_thread_data_t *tsm =
 187     vec_elt_at_index (sm->per_thread_data, thread_index);
 188
 189   if (is_fwd_bypass_session (s))
 190     {
 191       ed_key.l_addr = s->in2out.addr;
 192       ed_key.r_addr = s->ext_host_addr;
 193       ed_key.l_port = s->in2out.port;
 194       ed_key.r_port = s->ext_host_port;
 195       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 196       ed_key.fib_index = 0;
 197       ed_kv.key[0] = ed_key.as_u64[0];
 198       ed_kv.key[1] = ed_key.as_u64[1];
 199       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 200         nat_log_warn ("in2out_ed key del failed");
 201       return;
 202     }
 203
 204   /* session lookup tables */
 205   if (is_ed_session (s))
 206     {
 207       if (is_affinity_sessions (s))
 208         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 209                              s->in2out.protocol, s->out2in.port);
 210       ed_key.l_addr = s->out2in.addr;
 211       ed_key.r_addr = s->ext_host_addr;
 212       ed_key.fib_index = s->out2in.fib_index;
 213       if (snat_is_unk_proto_session (s))
 214         {
 215           ed_key.proto = s->in2out.port;
 216           ed_key.r_port = 0;
 217           ed_key.l_port = 0;
 218         }
 219       else
 220         {
 221           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 222           ed_key.l_port = s->out2in.port;
 223           ed_key.r_port = s->ext_host_port;
 224         }
 225       ed_kv.key[0] = ed_key.as_u64[0];
 226       ed_kv.key[1] = ed_key.as_u64[1];
 227       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 228         nat_log_warn ("out2in_ed key del failed");
 229       ed_key.l_addr = s->in2out.addr;
 230       ed_key.fib_index = s->in2out.fib_index;
 231       if (!snat_is_unk_proto_session (s))
 232         ed_key.l_port = s->in2out.port;
 233       if (is_twice_nat_session (s))
 234         {
 235           ed_key.r_addr = s->ext_host_nat_addr;
 236           ed_key.r_port = s->ext_host_nat_port;
 237         }
 238       ed_kv.key[0] = ed_key.as_u64[0];
 239       ed_kv.key[1] = ed_key.as_u64[1];
 240       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 241         nat_log_warn ("in2out_ed key del failed");
 242
 243       if (!is_ha)
 244         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 245                                &s->in2out.addr, s->in2out.port,
 246                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 247                                &s->out2in.addr, s->out2in.port,
 248                                &s->ext_host_addr, s->ext_host_port,
 249                                s->in2out.protocol, is_twice_nat_session (s));
 250     }
 251   else
 252     {
 253       kv.key = s->in2out.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 255         nat_log_warn ("in2out key del failed");
 256       kv.key = s->out2in.as_u64;
 257       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 258         nat_log_warn ("out2in key del failed");
 259
 260       if (!is_ha)
 261         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 262                                  &s->in2out.addr, s->in2out.port,
 263                                  &s->out2in.addr, s->out2in.port,
 264                                  s->in2out.protocol);
 265     }
 266
 267   if (snat_is_unk_proto_session (s))
 268     return;
 269
 270   if (!is_ha)
 271     {
 272       /* log NAT event */
 273       snat_ipfix_logging_nat44_ses_delete (thread_index,
 274                                            s->in2out.addr.as_u32,
 275                                            s->out2in.addr.as_u32,
 276                                            s->in2out.protocol,
 277                                            s->in2out.port,
 278                                            s->out2in.port,
 279                                            s->in2out.fib_index);
 280
 281       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 282                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 283                    thread_index);
 284     }
 285
 286   /* Twice NAT address and port for external host */
 287   if (is_twice_nat_session (s))
 288     {
 289       key.protocol = s->in2out.protocol;
 290       key.port = s->ext_host_nat_port;
 291       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 292       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 293                                           thread_index, &key);
 294     }
 295
 296   if (snat_is_session_static (s))
 297     return;
 298
 299   snat_free_outside_address_and_port (sm->addresses, thread_index,
 300                                       &s->out2in);
 301 }
 302
 303 snat_user_t *
 304 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 305                         u32 thread_index)
 306 {
 307   snat_user_t *u = 0;
 308   snat_user_key_t user_key;
 309   clib_bihash_kv_8_8_t kv, value;
 310   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 311   dlist_elt_t *per_user_list_head_elt;
 312
 313   user_key.addr.as_u32 = addr->as_u32;
 314   user_key.fib_index = fib_index;
 315   kv.key = user_key.as_u64;
 316
 317   /* Ever heard of the "user" = src ip4 address before? */
 318   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 319     {
 320       /* no, make a new one */
 321       pool_get (tsm->users, u);
 322       clib_memset (u, 0, sizeof (*u));
 323       u->addr.as_u32 = addr->as_u32;
 324       u->fib_index = fib_index;
 325
 326       pool_get (tsm->list_pool, per_user_list_head_elt);
 327
 328       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 329         tsm->list_pool;
 330
 331       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 332
 333       kv.value = u - tsm->users;
 334
 335       /* add user */
 336       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 337         nat_log_warn ("user_hash keay add failed");
 338
 339       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 340                                pool_elts (tsm->users));
 341     }
 342   else
 343     {
 344       u = pool_elt_at_index (tsm->users, value.value);
 345     }
 346
 347   return u;
 348 }
 349
 350 snat_session_t *
 351 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 352                               u32 thread_index, f64 now)
 353 {
 354   snat_session_t *s;
 355   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 356   u32 oldest_per_user_translation_list_index, session_index;
 357   dlist_elt_t *oldest_per_user_translation_list_elt;
 358   dlist_elt_t *per_user_translation_list_elt;
 359
 360   /* Over quota? Recycle the least recently used translation */
 361   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 362     {
 363       oldest_per_user_translation_list_index =
 364         clib_dlist_remove_head (tsm->list_pool,
 365                                 u->sessions_per_user_list_head_index);
 366
 367       ASSERT (oldest_per_user_translation_list_index != ~0);
 368
 369       /* Add it back to the end of the LRU list */
 370       clib_dlist_addtail (tsm->list_pool,
 371                           u->sessions_per_user_list_head_index,
 372                           oldest_per_user_translation_list_index);
 373       /* Get the list element */
 374       oldest_per_user_translation_list_elt =
 375         pool_elt_at_index (tsm->list_pool,
 376                            oldest_per_user_translation_list_index);
 377
 378       /* Get the session index from the list element */
 379       session_index = oldest_per_user_translation_list_elt->value;
 380
 381       /* Get the session */
 382       s = pool_elt_at_index (tsm->sessions, session_index);
 383       nat_free_session_data (sm, s, thread_index, 0);
 384       if (snat_is_session_static (s))
 385         u->nstaticsessions--;
 386       else
 387         u->nsessions--;
 388       s->flags = 0;
 389       s->total_bytes = 0;
 390       s->total_pkts = 0;
 391       s->state = 0;
 392       s->ext_host_addr.as_u32 = 0;
 393       s->ext_host_port = 0;
 394       s->ext_host_nat_addr.as_u32 = 0;
 395       s->ext_host_nat_port = 0;
 396     }
 397   else
 398     {
 399       pool_get (tsm->sessions, s);
 400       clib_memset (s, 0, sizeof (*s));
 401
 402       /* Create list elts */
 403       pool_get (tsm->list_pool, per_user_translation_list_elt);
 404       clib_dlist_init (tsm->list_pool,
 405                        per_user_translation_list_elt - tsm->list_pool);
 406
 407       per_user_translation_list_elt->value = s - tsm->sessions;
 408       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 409       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 410
 411       clib_dlist_addtail (tsm->list_pool,
 412                           s->per_user_list_head_index,
 413                           per_user_translation_list_elt - tsm->list_pool);
 414
 415       s->user_index = u - tsm->users;
 416       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 417                                pool_elts (tsm->sessions));
 418     }
 419
 420   s->ha_last_refreshed = now;
 421
 422   return s;
 423 }
 424
 425 snat_session_t *
 426 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 427                       f64 now)
 428 {
 429   snat_session_t *s;
 430   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 431   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 432   u32 oldest_index;
 433   u64 sess_timeout_time;
 434
 435   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 436     goto alloc_new;
 437
 438   oldest_index =
 439     clib_dlist_remove_head (tsm->list_pool,
 440                             u->sessions_per_user_list_head_index);
 441   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 442   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 443   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 444   if (now >= sess_timeout_time)
 445     {
 446       clib_dlist_addtail (tsm->list_pool,
 447                           u->sessions_per_user_list_head_index, oldest_index);
 448       nat_free_session_data (sm, s, thread_index, 0);
 449       if (snat_is_session_static (s))
 450         u->nstaticsessions--;
 451       else
 452         u->nsessions--;
 453       s->flags = 0;
 454       s->total_bytes = 0;
 455       s->total_pkts = 0;
 456       s->state = 0;
 457       s->ext_host_addr.as_u32 = 0;
 458       s->ext_host_port = 0;
 459       s->ext_host_nat_addr.as_u32 = 0;
 460       s->ext_host_nat_port = 0;
 461     }
 462   else
 463     {
 464       clib_dlist_addhead (tsm->list_pool,
 465                           u->sessions_per_user_list_head_index, oldest_index);
 466       if ((u->nsessions + u->nstaticsessions) >=
 467           sm->max_translations_per_user)
 468         {
 469           nat_log_warn ("max translations per user %U", format_ip4_address,
 470                         &u->addr);
 471           snat_ipfix_logging_max_entries_per_user
 472             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 473           return 0;
 474         }
 475       else
 476         {
 477         alloc_new:
 478           pool_get (tsm->sessions, s);
 479           clib_memset (s, 0, sizeof (*s));
 480
 481           /* Create list elts */
 482           pool_get (tsm->list_pool, per_user_translation_list_elt);
 483           clib_dlist_init (tsm->list_pool,
 484                            per_user_translation_list_elt - tsm->list_pool);
 485
 486           per_user_translation_list_elt->value = s - tsm->sessions;
 487           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 488           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 489
 490           clib_dlist_addtail (tsm->list_pool,
 491                               s->per_user_list_head_index,
 492                               per_user_translation_list_elt - tsm->list_pool);
 493         }
 494
 495       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 496                                pool_elts (tsm->sessions));
 497     }
 498
 499   s->ha_last_refreshed = now;
 500
 501   return s;
 502 }
 503
 504 void
 505 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 506                           int is_add)
 507 {
 508   fib_prefix_t prefix = {
 509     .fp_len = p_len,
 510     .fp_proto = FIB_PROTOCOL_IP4,
 511     .fp_addr = {
 512                 .ip4.as_u32 = addr->as_u32,
 513                 },
 514   };
 515   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 516
 517   if (is_add)
 518     fib_table_entry_update_one_path (fib_index,
 519                                      &prefix,
 520                                      FIB_SOURCE_PLUGIN_LOW,
 521                                      (FIB_ENTRY_FLAG_CONNECTED |
 522                                       FIB_ENTRY_FLAG_LOCAL |
 523                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 524                                      DPO_PROTO_IP4,
 525                                      NULL,
 526                                      sw_if_index,
 527                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 528   else
 529     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 530 }
 531
 532 int
 533 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 534                   u8 twice_nat)
 535 {
 536   snat_address_t *ap;
 537   snat_interface_t *i;
 538   vlib_thread_main_t *tm = vlib_get_thread_main ();
 539
 540   if (twice_nat && !sm->endpoint_dependent)
 541     return VNET_API_ERROR_FEATURE_DISABLED;
 542
 543   /* Check if address already exists */
 544   /* *INDENT-OFF* */
 545   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 546     {
 547       if (ap->addr.as_u32 == addr->as_u32)
 548         return VNET_API_ERROR_VALUE_EXIST;
 549     }
 550   /* *INDENT-ON* */
 551
 552   if (twice_nat)
 553     vec_add2 (sm->twice_nat_addresses, ap, 1);
 554   else
 555     vec_add2 (sm->addresses, ap, 1);
 556
 557   ap->addr = *addr;
 558   if (vrf_id != ~0)
 559     ap->fib_index =
 560       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 561                                          FIB_SOURCE_PLUGIN_LOW);
 562   else
 563     ap->fib_index = ~0;
 564 #define _(N, i, n, s) \
 565   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 566   ap->busy_##n##_ports = 0; \
 567   ap->busy_##n##_ports_per_thread = 0;\
 568   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 569   foreach_snat_protocol
 570 #undef _
 571     if (twice_nat)
 572     return 0;
 573
 574   /* Add external address to FIB */
 575   /* *INDENT-OFF* */
 576   pool_foreach (i, sm->interfaces,
 577   ({
 578     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 579       continue;
 580
 581     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 582     break;
 583   }));
 584   pool_foreach (i, sm->output_feature_interfaces,
 585   ({
 586     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 587       continue;
 588
 589     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 590     break;
 591   }));
 592   /* *INDENT-ON* */
 593
 594   return 0;
 595 }
 596
 597 static int
 598 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 599 {
 600   snat_static_mapping_t *m;
 601   /* *INDENT-OFF* */
 602   pool_foreach (m, sm->static_mappings,
 603   ({
 604       if (is_addr_only_static_mapping (m) ||
 605           is_out2in_only_static_mapping (m) ||
 606           is_identity_static_mapping (m))
 607         continue;
 608       if (m->external_addr.as_u32 == addr.as_u32)
 609         return 1;
 610   }));
 611   /* *INDENT-ON* */
 612
 613   return 0;
 614 }
 615
 616 void
 617 increment_v4_address (ip4_address_t * a)
 618 {
 619   u32 v;
 620
 621   v = clib_net_to_host_u32 (a->as_u32) + 1;
 622   a->as_u32 = clib_host_to_net_u32 (v);
 623 }
 624
 625 static void
 626 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 627                                        ip4_address_t l_addr,
 628                                        u16 l_port,
 629                                        u32 sw_if_index,
 630                                        u16 e_port,
 631                                        u32 vrf_id,
 632                                        snat_protocol_t proto,
 633                                        int addr_only, int is_add, u8 * tag,
 634                                        int twice_nat, int out2in_only,
 635                                        int identity_nat)
 636 {
 637   snat_static_map_resolve_t *rp;
 638
 639   vec_add2 (sm->to_resolve, rp, 1);
 640   rp->l_addr.as_u32 = l_addr.as_u32;
 641   rp->l_port = l_port;
 642   rp->sw_if_index = sw_if_index;
 643   rp->e_port = e_port;
 644   rp->vrf_id = vrf_id;
 645   rp->proto = proto;
 646   rp->addr_only = addr_only;
 647   rp->is_add = is_add;
 648   rp->twice_nat = twice_nat;
 649   rp->out2in_only = out2in_only;
 650   rp->identity_nat = identity_nat;
 651   rp->tag = vec_dup (tag);
 652 }
 653
 654 static u32
 655 get_thread_idx_by_port (u16 e_port)
 656 {
 657   snat_main_t *sm = &snat_main;
 658   u32 thread_idx = sm->num_workers;
 659   if (sm->num_workers > 1)
 660     {
 661       thread_idx =
 662         sm->first_worker_index +
 663         sm->workers[(e_port - 1024) / sm->port_per_thread];
 664     }
 665   return thread_idx;
 666 }
 667
 668 int
 669 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 670                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 671                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 672                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 673                          u8 identity_nat)
 674 {
 675   snat_main_t *sm = &snat_main;
 676   snat_static_mapping_t *m;
 677   snat_session_key_t m_key;
 678   clib_bihash_kv_8_8_t kv, value;
 679   snat_address_t *a = 0;
 680   u32 fib_index = ~0;
 681   snat_interface_t *interface;
 682   int i;
 683   snat_main_per_thread_data_t *tsm;
 684   snat_user_key_t u_key;
 685   snat_user_t *u;
 686   dlist_elt_t *head, *elt;
 687   u32 elt_index, head_index;
 688   u32 ses_index;
 689   u64 user_index;
 690   snat_session_t *s;
 691   snat_static_map_resolve_t *rp, *rp_match = 0;
 692   nat44_lb_addr_port_t *local;
 693   u32 find = ~0;
 694
 695   if (!sm->endpoint_dependent)
 696     {
 697       if (twice_nat || out2in_only)
 698         return VNET_API_ERROR_FEATURE_DISABLED;
 699     }
 700
 701   /* If the external address is a specific interface address */
 702   if (sw_if_index != ~0)
 703     {
 704       ip4_address_t *first_int_addr;
 705
 706       for (i = 0; i < vec_len (sm->to_resolve); i++)
 707         {
 708           rp = sm->to_resolve + i;
 709           if (rp->sw_if_index != sw_if_index ||
 710               rp->l_addr.as_u32 != l_addr.as_u32 ||
 711               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 712             continue;
 713
 714           if (!addr_only)
 715             {
 716               if (rp->l_port != l_port || rp->e_port != e_port
 717                   || rp->proto != proto)
 718                 continue;
 719             }
 720
 721           rp_match = rp;
 722           break;
 723         }
 724
 725       /* Might be already set... */
 726       first_int_addr = ip4_interface_first_address
 727         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 728
 729       if (is_add)
 730         {
 731           if (rp_match)
 732             return VNET_API_ERROR_VALUE_EXIST;
 733
 734           snat_add_static_mapping_when_resolved
 735             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 736              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 737
 738           /* DHCP resolution required? */
 739           if (first_int_addr == 0)
 740             {
 741               return 0;
 742             }
 743           else
 744             {
 745               e_addr.as_u32 = first_int_addr->as_u32;
 746               /* Identity mapping? */
 747               if (l_addr.as_u32 == 0)
 748                 l_addr.as_u32 = e_addr.as_u32;
 749             }
 750         }
 751       else
 752         {
 753           if (!rp_match)
 754             return VNET_API_ERROR_NO_SUCH_ENTRY;
 755
 756           vec_del1 (sm->to_resolve, i);
 757
 758           if (first_int_addr)
 759             {
 760               e_addr.as_u32 = first_int_addr->as_u32;
 761               /* Identity mapping? */
 762               if (l_addr.as_u32 == 0)
 763                 l_addr.as_u32 = e_addr.as_u32;
 764             }
 765           else
 766             return 0;
 767         }
 768     }
 769
 770   m_key.addr = e_addr;
 771   m_key.port = addr_only ? 0 : e_port;
 772   m_key.protocol = addr_only ? 0 : proto;
 773   m_key.fib_index = 0;
 774   kv.key = m_key.as_u64;
 775   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 776     m = 0;
 777   else
 778     m = pool_elt_at_index (sm->static_mappings, value.value);
 779
 780   if (is_add)
 781     {
 782       if (m)
 783         {
 784           if (is_identity_static_mapping (m))
 785             {
 786               /* *INDENT-OFF* */
 787               pool_foreach (local, m->locals,
 788               ({
 789                 if (local->vrf_id == vrf_id)
 790                   return VNET_API_ERROR_VALUE_EXIST;
 791               }));
 792               /* *INDENT-ON* */
 793               pool_get (m->locals, local);
 794               local->vrf_id = vrf_id;
 795               local->fib_index =
 796                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 797                                                    FIB_SOURCE_PLUGIN_LOW);
 798               m_key.addr = m->local_addr;
 799               m_key.port = m->local_port;
 800               m_key.protocol = m->proto;
 801               m_key.fib_index = local->fib_index;
 802               kv.key = m_key.as_u64;
 803               kv.value = m - sm->static_mappings;
 804               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 805               return 0;
 806             }
 807           else
 808             return VNET_API_ERROR_VALUE_EXIST;
 809         }
 810
 811       if (twice_nat && addr_only)
 812         return VNET_API_ERROR_UNSUPPORTED;
 813
 814       /* Convert VRF id to FIB index */
 815       if (vrf_id != ~0)
 816         fib_index =
 817           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 818                                              FIB_SOURCE_PLUGIN_LOW);
 819       /* If not specified use inside VRF id from SNAT plugin startup config */
 820       else
 821         {
 822           fib_index = sm->inside_fib_index;
 823           vrf_id = sm->inside_vrf_id;
 824         }
 825
 826       if (!(out2in_only || identity_nat))
 827         {
 828           m_key.addr = l_addr;
 829           m_key.port = addr_only ? 0 : l_port;
 830           m_key.protocol = addr_only ? 0 : proto;
 831           m_key.fib_index = fib_index;
 832           kv.key = m_key.as_u64;
 833           if (!clib_bihash_search_8_8
 834               (&sm->static_mapping_by_local, &kv, &value))
 835             return VNET_API_ERROR_VALUE_EXIST;
 836         }
 837
 838       /* Find external address in allocated addresses and reserve port for
 839          address and port pair mapping when dynamic translations enabled */
 840       if (!(addr_only || sm->static_mapping_only || out2in_only))
 841         {
 842           for (i = 0; i < vec_len (sm->addresses); i++)
 843             {
 844               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 845                 {
 846                   a = sm->addresses + i;
 847                   /* External port must be unused */
 848                   switch (proto)
 849                     {
 850 #define _(N, j, n, s) \
 851                     case SNAT_PROTOCOL_##N: \
 852                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 853                         return VNET_API_ERROR_INVALID_VALUE; \
 854                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 855                       if (e_port > 1024) \
 856                         { \
 857                           a->busy_##n##_ports++; \
 858                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 859                         } \
 860                       break;
 861                       foreach_snat_protocol
 862 #undef _
 863                     default:
 864                       nat_log_info ("unknown protocol");
 865                       return VNET_API_ERROR_INVALID_VALUE_2;
 866                     }
 867                   break;
 868                 }
 869             }
 870           /* External address must be allocated */
 871           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 872             {
 873               if (sw_if_index != ~0)
 874                 {
 875                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 876                     {
 877                       rp = sm->to_resolve + i;
 878                       if (rp->addr_only)
 879                         continue;
 880                       if (rp->sw_if_index != sw_if_index &&
 881                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 882                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 883                           rp->e_port != e_port && rp->proto != proto)
 884                         continue;
 885
 886                       vec_del1 (sm->to_resolve, i);
 887                       break;
 888                     }
 889                 }
 890               return VNET_API_ERROR_NO_SUCH_ENTRY;
 891             }
 892         }
 893
 894       pool_get (sm->static_mappings, m);
 895       clib_memset (m, 0, sizeof (*m));
 896       m->tag = vec_dup (tag);
 897       m->local_addr = l_addr;
 898       m->external_addr = e_addr;
 899       m->twice_nat = twice_nat;
 900       if (out2in_only)
 901         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 902       if (addr_only)
 903         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 904       if (identity_nat)
 905         {
 906           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 907           pool_get (m->locals, local);
 908           local->vrf_id = vrf_id;
 909           local->fib_index = fib_index;
 910         }
 911       else
 912         {
 913           m->vrf_id = vrf_id;
 914           m->fib_index = fib_index;
 915         }
 916       if (!addr_only)
 917         {
 918           m->local_port = l_port;
 919           m->external_port = e_port;
 920           m->proto = proto;
 921         }
 922
 923       if (sm->num_workers > 1)
 924         {
 925           ip4_header_t ip = {
 926             .src_address = m->local_addr,
 927           };
 928           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 929           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 930         }
 931       else
 932         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 933
 934       m_key.addr = m->local_addr;
 935       m_key.port = m->local_port;
 936       m_key.protocol = m->proto;
 937       m_key.fib_index = fib_index;
 938       kv.key = m_key.as_u64;
 939       kv.value = m - sm->static_mappings;
 940       if (!out2in_only)
 941         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 942
 943       m_key.addr = m->external_addr;
 944       m_key.port = m->external_port;
 945       m_key.fib_index = 0;
 946       kv.key = m_key.as_u64;
 947       kv.value = m - sm->static_mappings;
 948       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 949
 950       /* Delete dynamic sessions matching local address (+ local port) */
 951       if (!(sm->static_mapping_only))
 952         {
 953           u_key.addr = m->local_addr;
 954           u_key.fib_index = m->fib_index;
 955           kv.key = u_key.as_u64;
 956           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 957             {
 958               user_index = value.value;
 959               u = pool_elt_at_index (tsm->users, user_index);
 960               if (u->nsessions)
 961                 {
 962                   head_index = u->sessions_per_user_list_head_index;
 963                   head = pool_elt_at_index (tsm->list_pool, head_index);
 964                   elt_index = head->next;
 965                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 966                   ses_index = elt->value;
 967                   while (ses_index != ~0)
 968                     {
 969                       s = pool_elt_at_index (tsm->sessions, ses_index);
 970                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 971                       ses_index = elt->value;
 972
 973                       if (snat_is_session_static (s))
 974                         continue;
 975
 976                       if (!addr_only
 977                           && (clib_net_to_host_u16 (s->in2out.port) !=
 978                               m->local_port))
 979                         continue;
 980
 981                       nat_free_session_data (sm, s,
 982                                              tsm - sm->per_thread_data, 0);
 983                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 984
 985                       if (!addr_only && !sm->endpoint_dependent)
 986                         break;
 987                     }
 988                 }
 989             }
 990         }
 991     }
 992   else
 993     {
 994       if (!m)
 995         {
 996           if (sw_if_index != ~0)
 997             return 0;
 998           else
 999             return VNET_API_ERROR_NO_SUCH_ENTRY;
1000         }
1001
1002       if (identity_nat)
1003         {
1004           if (vrf_id == ~0)
1005             vrf_id = sm->inside_vrf_id;
1006
1007           /* *INDENT-OFF* */
1008           pool_foreach (local, m->locals,
1009           ({
1010             if (local->vrf_id == vrf_id)
1011               find = local - m->locals;
1012           }));
1013           /* *INDENT-ON* */
1014           if (find == ~0)
1015             return VNET_API_ERROR_NO_SUCH_ENTRY;
1016
1017           local = pool_elt_at_index (m->locals, find);
1018           fib_index = local->fib_index;
1019           pool_put (m->locals, local);
1020         }
1021       else
1022         fib_index = m->fib_index;
1023
1024       /* Free external address port */
1025       if (!(addr_only || sm->static_mapping_only || out2in_only))
1026         {
1027           for (i = 0; i < vec_len (sm->addresses); i++)
1028             {
1029               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1030                 {
1031                   a = sm->addresses + i;
1032                   switch (proto)
1033                     {
1034 #define _(N, j, n, s) \
1035                     case SNAT_PROTOCOL_##N: \
1036                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1037                       if (e_port > 1024) \
1038                         { \
1039                           a->busy_##n##_ports--; \
1040                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1041                         } \
1042                       break;
1043                       foreach_snat_protocol
1044 #undef _
1045                     default:
1046                       nat_log_info ("unknown protocol");
1047                       return VNET_API_ERROR_INVALID_VALUE_2;
1048                     }
1049                   break;
1050                 }
1051             }
1052         }
1053
1054       if (sm->num_workers > 1)
1055         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1056       else
1057         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1058
1059       m_key.addr = m->local_addr;
1060       m_key.port = m->local_port;
1061       m_key.protocol = m->proto;
1062       m_key.fib_index = fib_index;
1063       kv.key = m_key.as_u64;
1064       if (!out2in_only)
1065         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1066
1067       /* Delete session(s) for static mapping if exist */
1068       if (!(sm->static_mapping_only) ||
1069           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1070         {
1071           u_key.addr = m->local_addr;
1072           u_key.fib_index = fib_index;
1073           kv.key = u_key.as_u64;
1074           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1075             {
1076               user_index = value.value;
1077               u = pool_elt_at_index (tsm->users, user_index);
1078               if (u->nstaticsessions)
1079                 {
1080                   head_index = u->sessions_per_user_list_head_index;
1081                   head = pool_elt_at_index (tsm->list_pool, head_index);
1082                   elt_index = head->next;
1083                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1084                   ses_index = elt->value;
1085                   while (ses_index != ~0)
1086                     {
1087                       s = pool_elt_at_index (tsm->sessions, ses_index);
1088                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1089                       ses_index = elt->value;
1090
1091                       if (!addr_only)
1092                         {
1093                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1094                               (clib_net_to_host_u16 (s->out2in.port) !=
1095                                e_port))
1096                             continue;
1097                         }
1098
1099                       if (is_lb_session (s))
1100                         continue;
1101
1102                       if (!snat_is_session_static (s))
1103                         continue;
1104
1105                       nat_free_session_data (sm, s,
1106                                              tsm - sm->per_thread_data, 0);
1107                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1108
1109                       if (!addr_only && !sm->endpoint_dependent)
1110                         break;
1111                     }
1112                 }
1113             }
1114         }
1115
1116       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1117       if (pool_elts (m->locals))
1118         return 0;
1119
1120       m_key.addr = m->external_addr;
1121       m_key.port = m->external_port;
1122       m_key.fib_index = 0;
1123       kv.key = m_key.as_u64;
1124       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1125
1126       vec_free (m->tag);
1127       vec_free (m->workers);
1128       /* Delete static mapping from pool */
1129       pool_put (sm->static_mappings, m);
1130     }
1131
1132   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1133     return 0;
1134
1135   /* Add/delete external address to FIB */
1136   /* *INDENT-OFF* */
1137   pool_foreach (interface, sm->interfaces,
1138   ({
1139     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1140       continue;
1141
1142     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1143     break;
1144   }));
1145   pool_foreach (interface, sm->output_feature_interfaces,
1146   ({
1147     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1148       continue;
1149
1150     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1151     break;
1152   }));
1153   /* *INDENT-ON* */
1154
1155   return 0;
1156 }
1157
1158 int
1159 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1160                                  snat_protocol_t proto,
1161                                  nat44_lb_addr_port_t * locals, u8 is_add,
1162                                  twice_nat_type_t twice_nat, u8 out2in_only,
1163                                  u8 * tag, u32 affinity)
1164 {
1165   snat_main_t *sm = &snat_main;
1166   snat_static_mapping_t *m;
1167   snat_session_key_t m_key;
1168   clib_bihash_kv_8_8_t kv, value;
1169   snat_address_t *a = 0;
1170   int i;
1171   nat44_lb_addr_port_t *local;
1172   u32 elt_index, head_index, ses_index;
1173   snat_main_per_thread_data_t *tsm;
1174   snat_user_key_t u_key;
1175   snat_user_t *u;
1176   snat_session_t *s;
1177   dlist_elt_t *head, *elt;
1178   uword *bitmap = 0;
1179
1180   if (!sm->endpoint_dependent)
1181     return VNET_API_ERROR_FEATURE_DISABLED;
1182
1183   m_key.addr = e_addr;
1184   m_key.port = e_port;
1185   m_key.protocol = proto;
1186   m_key.fib_index = 0;
1187   kv.key = m_key.as_u64;
1188   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1189     m = 0;
1190   else
1191     m = pool_elt_at_index (sm->static_mappings, value.value);
1192
1193   if (is_add)
1194     {
1195       if (m)
1196         return VNET_API_ERROR_VALUE_EXIST;
1197
1198       if (vec_len (locals) < 2)
1199         return VNET_API_ERROR_INVALID_VALUE;
1200
1201       /* Find external address in allocated addresses and reserve port for
1202          address and port pair mapping when dynamic translations enabled */
1203       if (!(sm->static_mapping_only || out2in_only))
1204         {
1205           for (i = 0; i < vec_len (sm->addresses); i++)
1206             {
1207               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1208                 {
1209                   a = sm->addresses + i;
1210                   /* External port must be unused */
1211                   switch (proto)
1212                     {
1213 #define _(N, j, n, s) \
1214                     case SNAT_PROTOCOL_##N: \
1215                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1216                         return VNET_API_ERROR_INVALID_VALUE; \
1217                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1218                       if (e_port > 1024) \
1219                         { \
1220                           a->busy_##n##_ports++; \
1221                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1222                         } \
1223                       break;
1224                       foreach_snat_protocol
1225 #undef _
1226                     default:
1227                       nat_log_info ("unknown protocol");
1228                       return VNET_API_ERROR_INVALID_VALUE_2;
1229                     }
1230                   break;
1231                 }
1232             }
1233           /* External address must be allocated */
1234           if (!a)
1235             return VNET_API_ERROR_NO_SUCH_ENTRY;
1236         }
1237
1238       pool_get (sm->static_mappings, m);
1239       clib_memset (m, 0, sizeof (*m));
1240       m->tag = vec_dup (tag);
1241       m->external_addr = e_addr;
1242       m->external_port = e_port;
1243       m->proto = proto;
1244       m->twice_nat = twice_nat;
1245       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1246       if (out2in_only)
1247         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1248       m->affinity = affinity;
1249
1250       if (affinity)
1251         m->affinity_per_service_list_head_index =
1252           nat_affinity_get_per_service_list_head_index ();
1253       else
1254         m->affinity_per_service_list_head_index = ~0;
1255
1256       m_key.addr = m->external_addr;
1257       m_key.port = m->external_port;
1258       m_key.protocol = m->proto;
1259       m_key.fib_index = 0;
1260       kv.key = m_key.as_u64;
1261       kv.value = m - sm->static_mappings;
1262       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1263         {
1264           nat_log_err ("static_mapping_by_external key add failed");
1265           return VNET_API_ERROR_UNSPECIFIED;
1266         }
1267
1268       m_key.fib_index = m->fib_index;
1269       for (i = 0; i < vec_len (locals); i++)
1270         {
1271           locals[i].fib_index =
1272             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1273                                                locals[i].vrf_id,
1274                                                FIB_SOURCE_PLUGIN_LOW);
1275           m_key.addr = locals[i].addr;
1276           m_key.fib_index = locals[i].fib_index;
1277           if (!out2in_only)
1278             {
1279               m_key.port = locals[i].port;
1280               kv.key = m_key.as_u64;
1281               kv.value = m - sm->static_mappings;
1282               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1283             }
1284           locals[i].prefix = (i == 0) ? locals[i].probability :
1285             (locals[i - 1].prefix + locals[i].probability);
1286           pool_get (m->locals, local);
1287           *local = locals[i];
1288           if (sm->num_workers > 1)
1289             {
1290               ip4_header_t ip = {
1291                 .src_address = locals[i].addr,
1292               };
1293               bitmap =
1294                 clib_bitmap_set (bitmap,
1295                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1296             }
1297         }
1298
1299       /* Assign workers */
1300       if (sm->num_workers > 1)
1301         {
1302           /* *INDENT-OFF* */
1303           clib_bitmap_foreach (i, bitmap,
1304             ({
1305                vec_add1(m->workers, i);
1306             }));
1307           /* *INDENT-ON* */
1308         }
1309     }
1310   else
1311     {
1312       if (!m)
1313         return VNET_API_ERROR_NO_SUCH_ENTRY;
1314
1315       if (!is_lb_static_mapping (m))
1316         return VNET_API_ERROR_INVALID_VALUE;
1317
1318       /* Free external address port */
1319       if (!(sm->static_mapping_only || out2in_only))
1320         {
1321           for (i = 0; i < vec_len (sm->addresses); i++)
1322             {
1323               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1324                 {
1325                   a = sm->addresses + i;
1326                   switch (proto)
1327                     {
1328 #define _(N, j, n, s) \
1329                     case SNAT_PROTOCOL_##N: \
1330                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1331                       if (e_port > 1024) \
1332                         { \
1333                           a->busy_##n##_ports--; \
1334                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1335                         } \
1336                       break;
1337                       foreach_snat_protocol
1338 #undef _
1339                     default:
1340                       nat_log_info ("unknown protocol");
1341                       return VNET_API_ERROR_INVALID_VALUE_2;
1342                     }
1343                   break;
1344                 }
1345             }
1346         }
1347
1348       m_key.addr = m->external_addr;
1349       m_key.port = m->external_port;
1350       m_key.protocol = m->proto;
1351       m_key.fib_index = 0;
1352       kv.key = m_key.as_u64;
1353       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1354         {
1355           nat_log_err ("static_mapping_by_external key del failed");
1356           return VNET_API_ERROR_UNSPECIFIED;
1357         }
1358
1359       /* *INDENT-OFF* */
1360       pool_foreach (local, m->locals,
1361       ({
1362           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1363                             FIB_SOURCE_PLUGIN_LOW);
1364           m_key.addr = local->addr;
1365           if (!out2in_only)
1366             {
1367               m_key.port = local->port;
1368               m_key.fib_index = local->fib_index;
1369               kv.key = m_key.as_u64;
1370               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1371                 {
1372                   nat_log_err ("static_mapping_by_local key del failed");
1373                   return VNET_API_ERROR_UNSPECIFIED;
1374                 }
1375             }
1376
1377           if (sm->num_workers > 1)
1378             {
1379               ip4_header_t ip = {
1380                 .src_address = local->addr,
1381               };
1382               tsm = vec_elt_at_index (sm->per_thread_data,
1383                                       sm->worker_in2out_cb (&ip, m->fib_index));
1384             }
1385           else
1386             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1387
1388           /* Delete sessions */
1389           u_key.addr = local->addr;
1390           u_key.fib_index = local->fib_index;
1391           kv.key = u_key.as_u64;
1392           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1393             {
1394               u = pool_elt_at_index (tsm->users, value.value);
1395               if (u->nstaticsessions)
1396                 {
1397                   head_index = u->sessions_per_user_list_head_index;
1398                   head = pool_elt_at_index (tsm->list_pool, head_index);
1399                   elt_index = head->next;
1400                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1401                   ses_index = elt->value;
1402                   while (ses_index != ~0)
1403                     {
1404                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1405                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1406                       ses_index = elt->value;
1407
1408                       if (!(is_lb_session (s)))
1409                         continue;
1410
1411                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1412                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1413                         continue;
1414
1415                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1416                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1417                     }
1418                 }
1419             }
1420       }));
1421       /* *INDENT-ON* */
1422       if (m->affinity)
1423         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1424       pool_free (m->locals);
1425       vec_free (m->tag);
1426       vec_free (m->workers);
1427
1428       pool_put (sm->static_mappings, m);
1429     }
1430
1431   return 0;
1432 }
1433
1434 int
1435 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1436                                        ip4_address_t l_addr, u16 l_port,
1437                                        snat_protocol_t proto, u32 vrf_id,
1438                                        u8 probability, u8 is_add)
1439 {
1440   snat_main_t *sm = &snat_main;
1441   snat_static_mapping_t *m = 0;
1442   snat_session_key_t m_key;
1443   clib_bihash_kv_8_8_t kv, value;
1444   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1445   snat_main_per_thread_data_t *tsm;
1446   snat_user_key_t u_key;
1447   snat_user_t *u;
1448   snat_session_t *s;
1449   dlist_elt_t *head, *elt;
1450   u32 elt_index, head_index, ses_index, *locals = 0;
1451   uword *bitmap = 0;
1452   int i;
1453
1454   if (!sm->endpoint_dependent)
1455     return VNET_API_ERROR_FEATURE_DISABLED;
1456
1457   m_key.addr = e_addr;
1458   m_key.port = e_port;
1459   m_key.protocol = proto;
1460   m_key.fib_index = 0;
1461   kv.key = m_key.as_u64;
1462   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1463     m = pool_elt_at_index (sm->static_mappings, value.value);
1464
1465   if (!m)
1466     return VNET_API_ERROR_NO_SUCH_ENTRY;
1467
1468   if (!is_lb_static_mapping (m))
1469     return VNET_API_ERROR_INVALID_VALUE;
1470
1471   /* *INDENT-OFF* */
1472   pool_foreach (local, m->locals,
1473   ({
1474     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1475         (local->vrf_id == vrf_id))
1476       {
1477         match_local = local;
1478         break;
1479       }
1480   }));
1481   /* *INDENT-ON* */
1482
1483   if (is_add)
1484     {
1485       if (match_local)
1486         return VNET_API_ERROR_VALUE_EXIST;
1487
1488       pool_get (m->locals, local);
1489       clib_memset (local, 0, sizeof (*local));
1490       local->addr.as_u32 = l_addr.as_u32;
1491       local->port = l_port;
1492       local->probability = probability;
1493       local->vrf_id = vrf_id;
1494       local->fib_index =
1495         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1496                                            FIB_SOURCE_PLUGIN_LOW);
1497
1498       if (!is_out2in_only_static_mapping (m))
1499         {
1500           m_key.addr = l_addr;
1501           m_key.port = l_port;
1502           m_key.fib_index = local->fib_index;
1503           kv.key = m_key.as_u64;
1504           kv.value = m - sm->static_mappings;
1505           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1506             nat_log_err ("static_mapping_by_local key add failed");
1507         }
1508     }
1509   else
1510     {
1511       if (!match_local)
1512         return VNET_API_ERROR_NO_SUCH_ENTRY;
1513
1514       if (pool_elts (m->locals) < 3)
1515         return VNET_API_ERROR_UNSPECIFIED;
1516
1517       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1518                         FIB_SOURCE_PLUGIN_LOW);
1519
1520       if (!is_out2in_only_static_mapping (m))
1521         {
1522           m_key.addr = l_addr;
1523           m_key.port = l_port;
1524           m_key.fib_index = match_local->fib_index;
1525           kv.key = m_key.as_u64;
1526           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1527             nat_log_err ("static_mapping_by_local key del failed");
1528         }
1529
1530       if (sm->num_workers > 1)
1531         {
1532           ip4_header_t ip = {
1533             .src_address = local->addr,
1534           };
1535           tsm = vec_elt_at_index (sm->per_thread_data,
1536                                   sm->worker_in2out_cb (&ip, m->fib_index));
1537         }
1538       else
1539         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1540
1541       /* Delete sessions */
1542       u_key.addr = match_local->addr;
1543       u_key.fib_index = match_local->fib_index;
1544       kv.key = u_key.as_u64;
1545       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1546         {
1547           u = pool_elt_at_index (tsm->users, value.value);
1548           if (u->nstaticsessions)
1549             {
1550               head_index = u->sessions_per_user_list_head_index;
1551               head = pool_elt_at_index (tsm->list_pool, head_index);
1552               elt_index = head->next;
1553               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1554               ses_index = elt->value;
1555               while (ses_index != ~0)
1556                 {
1557                   s = pool_elt_at_index (tsm->sessions, ses_index);
1558                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1559                   ses_index = elt->value;
1560
1561                   if (!(is_lb_session (s)))
1562                     continue;
1563
1564                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1565                       (clib_net_to_host_u16 (s->in2out.port) !=
1566                        match_local->port))
1567                     continue;
1568
1569                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1570                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1571                 }
1572             }
1573         }
1574
1575       pool_put (m->locals, match_local);
1576     }
1577
1578   vec_free (m->workers);
1579
1580   /* *INDENT-OFF* */
1581   pool_foreach (local, m->locals,
1582   ({
1583     vec_add1 (locals, local - m->locals);
1584     if (sm->num_workers > 1)
1585       {
1586         ip4_header_t ip;
1587         ip.src_address.as_u32 = local->addr.as_u32,
1588         bitmap = clib_bitmap_set (bitmap,
1589                                   sm->worker_in2out_cb (&ip, local->fib_index),
1590                                   1);
1591       }
1592   }));
1593   /* *INDENT-ON* */
1594
1595   ASSERT (vec_len (locals) > 1);
1596
1597   local = pool_elt_at_index (m->locals, locals[0]);
1598   local->prefix = local->probability;
1599   for (i = 1; i < vec_len (locals); i++)
1600     {
1601       local = pool_elt_at_index (m->locals, locals[i]);
1602       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1603       local->prefix = local->probability + prev_local->prefix;
1604     }
1605
1606   /* Assign workers */
1607   if (sm->num_workers > 1)
1608     {
1609       /* *INDENT-OFF* */
1610       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1611       /* *INDENT-ON* */
1612     }
1613
1614   return 0;
1615 }
1616
1617 int
1618 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1619                   u8 twice_nat)
1620 {
1621   snat_address_t *a = 0;
1622   snat_session_t *ses;
1623   u32 *ses_to_be_removed = 0, *ses_index;
1624   snat_main_per_thread_data_t *tsm;
1625   snat_static_mapping_t *m;
1626   snat_interface_t *interface;
1627   int i;
1628   snat_address_t *addresses =
1629     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1630
1631   /* Find SNAT address */
1632   for (i = 0; i < vec_len (addresses); i++)
1633     {
1634       if (addresses[i].addr.as_u32 == addr.as_u32)
1635         {
1636           a = addresses + i;
1637           break;
1638         }
1639     }
1640   if (!a)
1641     return VNET_API_ERROR_NO_SUCH_ENTRY;
1642
1643   if (delete_sm)
1644     {
1645       /* *INDENT-OFF* */
1646       pool_foreach (m, sm->static_mappings,
1647       ({
1648           if (m->external_addr.as_u32 == addr.as_u32)
1649             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1650                                             m->local_port, m->external_port,
1651                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1652                                             m->proto, 0, m->twice_nat,
1653                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1654       }));
1655       /* *INDENT-ON* */
1656     }
1657   else
1658     {
1659       /* Check if address is used in some static mapping */
1660       if (is_snat_address_used_in_static_mapping (sm, addr))
1661         {
1662           nat_log_notice ("address used in static mapping");
1663           return VNET_API_ERROR_UNSPECIFIED;
1664         }
1665     }
1666
1667   if (a->fib_index != ~0)
1668     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1669
1670   /* Delete sessions using address */
1671   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1672     {
1673       /* *INDENT-OFF* */
1674       vec_foreach (tsm, sm->per_thread_data)
1675         {
1676           pool_foreach (ses, tsm->sessions, ({
1677             if (ses->out2in.addr.as_u32 == addr.as_u32)
1678               {
1679                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1680                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1681               }
1682           }));
1683
1684           vec_foreach (ses_index, ses_to_be_removed)
1685             {
1686               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1687               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1688             }
1689
1690           vec_free (ses_to_be_removed);
1691         }
1692       /* *INDENT-ON* */
1693     }
1694
1695 #define _(N, i, n, s) \
1696   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1697   vec_free (a->busy_##n##_ports_per_thread);
1698   foreach_snat_protocol
1699 #undef _
1700     if (twice_nat)
1701     {
1702       vec_del1 (sm->twice_nat_addresses, i);
1703       return 0;
1704     }
1705   else
1706     vec_del1 (sm->addresses, i);
1707
1708   /* Delete external address from FIB */
1709   /* *INDENT-OFF* */
1710   pool_foreach (interface, sm->interfaces,
1711   ({
1712     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1713       continue;
1714
1715     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1716     break;
1717   }));
1718   pool_foreach (interface, sm->output_feature_interfaces,
1719   ({
1720     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1721       continue;
1722
1723     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1724     break;
1725   }));
1726   /* *INDENT-ON* */
1727
1728   return 0;
1729 }
1730
1731 int
1732 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1733 {
1734   snat_main_t *sm = &snat_main;
1735   snat_interface_t *i;
1736   const char *feature_name, *del_feature_name;
1737   snat_address_t *ap;
1738   snat_static_mapping_t *m;
1739   snat_det_map_t *dm;
1740   nat_outside_fib_t *outside_fib;
1741   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1742                                                        sw_if_index);
1743
1744   if (sm->out2in_dpo && !is_inside)
1745     return VNET_API_ERROR_UNSUPPORTED;
1746
1747   /* *INDENT-OFF* */
1748   pool_foreach (i, sm->output_feature_interfaces,
1749   ({
1750     if (i->sw_if_index == sw_if_index)
1751       return VNET_API_ERROR_VALUE_EXIST;
1752   }));
1753   /* *INDENT-ON* */
1754
1755   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1756     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1757   else
1758     {
1759       if (sm->num_workers > 1 && !sm->deterministic)
1760         feature_name =
1761           is_inside ? "nat44-in2out-worker-handoff" :
1762           "nat44-out2in-worker-handoff";
1763       else if (sm->deterministic)
1764         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1765       else if (sm->endpoint_dependent)
1766         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1767       else
1768         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1769     }
1770
1771   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1772     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1773                                                       NAT_FQ_NELTS);
1774
1775   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1776     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1777                                                       NAT_FQ_NELTS);
1778
1779   if (!is_inside)
1780     {
1781       /* *INDENT-OFF* */
1782       vec_foreach (outside_fib, sm->outside_fibs)
1783         {
1784           if (outside_fib->fib_index == fib_index)
1785             {
1786               if (is_del)
1787                 {
1788                   outside_fib->refcount--;
1789                   if (!outside_fib->refcount)
1790                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1791                 }
1792               else
1793                 outside_fib->refcount++;
1794               goto feature_set;
1795             }
1796         }
1797       /* *INDENT-ON* */
1798       if (!is_del)
1799         {
1800           vec_add2 (sm->outside_fibs, outside_fib, 1);
1801           outside_fib->refcount = 1;
1802           outside_fib->fib_index = fib_index;
1803         }
1804     }
1805 feature_set:
1806   /* *INDENT-OFF* */
1807   pool_foreach (i, sm->interfaces,
1808   ({
1809     if (i->sw_if_index == sw_if_index)
1810       {
1811         if (is_del)
1812           {
1813             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1814               {
1815                 if (is_inside)
1816                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1817                 else
1818                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1819
1820                 if (sm->num_workers > 1 && !sm->deterministic)
1821                   {
1822                     del_feature_name = "nat44-handoff-classify";
1823                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1824                                                  "nat44-out2in-worker-handoff";
1825                   }
1826                 else if (sm->deterministic)
1827                   {
1828                     del_feature_name = "nat44-det-classify";
1829                     feature_name = !is_inside ?  "nat44-det-in2out" :
1830                                                  "nat44-det-out2in";
1831                   }
1832                 else if (sm->endpoint_dependent)
1833                   {
1834                     del_feature_name = "nat44-ed-classify";
1835                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1836                                                  "nat44-ed-out2in";
1837                   }
1838                 else
1839                   {
1840                     del_feature_name = "nat44-classify";
1841                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1842                   }
1843
1844                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1845                                              sw_if_index, 0, 0, 0);
1846                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1847                                              sw_if_index, 1, 0, 0);
1848                 if (!is_inside)
1849                   {
1850                     if (sm->endpoint_dependent)
1851                       vnet_feature_enable_disable ("ip4-local",
1852                                                    "nat44-ed-hairpinning",
1853                                                    sw_if_index, 1, 0, 0);
1854                     else if (!sm->deterministic)
1855                       vnet_feature_enable_disable ("ip4-local",
1856                                                    "nat44-hairpinning",
1857                                                    sw_if_index, 1, 0, 0);
1858                   }
1859               }
1860             else
1861               {
1862                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1863                                              sw_if_index, 0, 0, 0);
1864                 pool_put (sm->interfaces, i);
1865                 if (is_inside)
1866                   {
1867                     if (sm->endpoint_dependent)
1868                       vnet_feature_enable_disable ("ip4-local",
1869                                                    "nat44-ed-hairpinning",
1870                                                    sw_if_index, 0, 0, 0);
1871                     else if (!sm->deterministic)
1872                       vnet_feature_enable_disable ("ip4-local",
1873                                                    "nat44-hairpinning",
1874                                                    sw_if_index, 0, 0, 0);
1875                   }
1876               }
1877           }
1878         else
1879           {
1880             if ((nat_interface_is_inside(i) && is_inside) ||
1881                 (nat_interface_is_outside(i) && !is_inside))
1882               return 0;
1883
1884             if (sm->num_workers > 1 && !sm->deterministic)
1885               {
1886                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1887                                                  "nat44-out2in-worker-handoff";
1888                 feature_name = "nat44-handoff-classify";
1889               }
1890             else if (sm->deterministic)
1891               {
1892                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1893                                                  "nat44-det-out2in";
1894                 feature_name = "nat44-det-classify";
1895               }
1896             else if (sm->endpoint_dependent)
1897               {
1898                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1899                                                  "nat44-ed-out2in";
1900                 feature_name = "nat44-ed-classify";
1901               }
1902             else
1903               {
1904                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1905                 feature_name = "nat44-classify";
1906               }
1907
1908             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1909                                          sw_if_index, 0, 0, 0);
1910             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1911                                          sw_if_index, 1, 0, 0);
1912             if (!is_inside)
1913               {
1914                 if (sm->endpoint_dependent)
1915                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1916                                                sw_if_index, 0, 0, 0);
1917                 else if (!sm->deterministic)
1918                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1919                                                sw_if_index, 0, 0, 0);
1920               }
1921             goto set_flags;
1922           }
1923
1924         goto fib;
1925       }
1926   }));
1927   /* *INDENT-ON* */
1928
1929   if (is_del)
1930     return VNET_API_ERROR_NO_SUCH_ENTRY;
1931
1932   pool_get (sm->interfaces, i);
1933   i->sw_if_index = sw_if_index;
1934   i->flags = 0;
1935   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1936                                0);
1937
1938   if (is_inside && !sm->out2in_dpo)
1939     {
1940       if (sm->endpoint_dependent)
1941         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1942                                      sw_if_index, 1, 0, 0);
1943       else if (!sm->deterministic)
1944         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1945                                      sw_if_index, 1, 0, 0);
1946     }
1947
1948 set_flags:
1949   if (is_inside)
1950     {
1951       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1952       return 0;
1953     }
1954   else
1955     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1956
1957   /* Add/delete external addresses to FIB */
1958 fib:
1959   /* *INDENT-OFF* */
1960   vec_foreach (ap, sm->addresses)
1961     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1962
1963   pool_foreach (m, sm->static_mappings,
1964   ({
1965     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1966       continue;
1967
1968     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1969   }));
1970
1971   pool_foreach (dm, sm->det_maps,
1972   ({
1973     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1974   }));
1975   /* *INDENT-ON* */
1976
1977   return 0;
1978 }
1979
1980 int
1981 snat_interface_add_del_output_feature (u32 sw_if_index,
1982                                        u8 is_inside, int is_del)
1983 {
1984   snat_main_t *sm = &snat_main;
1985   snat_interface_t *i;
1986   snat_address_t *ap;
1987   snat_static_mapping_t *m;
1988   nat_outside_fib_t *outside_fib;
1989   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1990                                                        sw_if_index);
1991
1992
1993   if (sm->deterministic ||
1994       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1995     return VNET_API_ERROR_UNSUPPORTED;
1996
1997   /* *INDENT-OFF* */
1998   pool_foreach (i, sm->interfaces,
1999   ({
2000     if (i->sw_if_index == sw_if_index)
2001       return VNET_API_ERROR_VALUE_EXIST;
2002   }));
2003   /* *INDENT-ON* */
2004
2005   if (!is_inside)
2006     {
2007       /* *INDENT-OFF* */
2008       vec_foreach (outside_fib, sm->outside_fibs)
2009         {
2010           if (outside_fib->fib_index == fib_index)
2011             {
2012               if (is_del)
2013                 {
2014                   outside_fib->refcount--;
2015                   if (!outside_fib->refcount)
2016                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2017                 }
2018               else
2019                 outside_fib->refcount++;
2020               goto feature_set;
2021             }
2022         }
2023       /* *INDENT-ON* */
2024       if (!is_del)
2025         {
2026           vec_add2 (sm->outside_fibs, outside_fib, 1);
2027           outside_fib->refcount = 1;
2028           outside_fib->fib_index = fib_index;
2029         }
2030     }
2031
2032 feature_set:
2033   if (is_inside)
2034     {
2035       if (sm->endpoint_dependent)
2036         {
2037           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2038                                        sw_if_index, !is_del, 0, 0);
2039           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2040                                        sw_if_index, !is_del, 0, 0);
2041         }
2042       else
2043         {
2044           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2045                                        sw_if_index, !is_del, 0, 0);
2046           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2047                                        sw_if_index, !is_del, 0, 0);
2048         }
2049       goto fq;
2050     }
2051
2052   if (sm->num_workers > 1)
2053     {
2054       vnet_feature_enable_disable ("ip4-unicast",
2055                                    "nat44-out2in-worker-handoff",
2056                                    sw_if_index, !is_del, 0, 0);
2057       vnet_feature_enable_disable ("ip4-output",
2058                                    "nat44-in2out-output-worker-handoff",
2059                                    sw_if_index, !is_del, 0, 0);
2060     }
2061   else
2062     {
2063       if (sm->endpoint_dependent)
2064         {
2065           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2066                                        sw_if_index, !is_del, 0, 0);
2067           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2068                                        sw_if_index, !is_del, 0, 0);
2069         }
2070       else
2071         {
2072           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2073                                        sw_if_index, !is_del, 0, 0);
2074           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2075                                        sw_if_index, !is_del, 0, 0);
2076         }
2077     }
2078
2079 fq:
2080   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2081     sm->fq_in2out_output_index =
2082       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2083
2084   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2085     sm->fq_out2in_index =
2086       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2087
2088   /* *INDENT-OFF* */
2089   pool_foreach (i, sm->output_feature_interfaces,
2090   ({
2091     if (i->sw_if_index == sw_if_index)
2092       {
2093         if (is_del)
2094           pool_put (sm->output_feature_interfaces, i);
2095         else
2096           return VNET_API_ERROR_VALUE_EXIST;
2097
2098         goto fib;
2099       }
2100   }));
2101   /* *INDENT-ON* */
2102
2103   if (is_del)
2104     return VNET_API_ERROR_NO_SUCH_ENTRY;
2105
2106   pool_get (sm->output_feature_interfaces, i);
2107   i->sw_if_index = sw_if_index;
2108   i->flags = 0;
2109   if (is_inside)
2110     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2111   else
2112     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2113
2114   /* Add/delete external addresses to FIB */
2115 fib:
2116   if (is_inside)
2117     return 0;
2118
2119   /* *INDENT-OFF* */
2120   vec_foreach (ap, sm->addresses)
2121     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2122
2123   pool_foreach (m, sm->static_mappings,
2124   ({
2125     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2126       continue;
2127
2128     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2129   }));
2130   /* *INDENT-ON* */
2131
2132   return 0;
2133 }
2134
2135 int
2136 snat_set_workers (uword * bitmap)
2137 {
2138   snat_main_t *sm = &snat_main;
2139   int i, j = 0;
2140
2141   if (sm->num_workers < 2)
2142     return VNET_API_ERROR_FEATURE_DISABLED;
2143
2144   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2145     return VNET_API_ERROR_INVALID_WORKER;
2146
2147   vec_free (sm->workers);
2148   /* *INDENT-OFF* */
2149   clib_bitmap_foreach (i, bitmap,
2150     ({
2151       vec_add1(sm->workers, i);
2152       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2153       j++;
2154     }));
2155   /* *INDENT-ON* */
2156
2157   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2158   sm->num_snat_thread = _vec_len (sm->workers);
2159
2160   return 0;
2161 }
2162
2163 static void
2164 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2165                          u32 old_fib_index)
2166 {
2167   snat_main_t *sm = &snat_main;
2168   nat_outside_fib_t *outside_fib;
2169   snat_interface_t *i;
2170   u8 is_add = 1;
2171
2172   if (new_fib_index == old_fib_index)
2173     return;
2174
2175   if (!vec_len (sm->outside_fibs))
2176     return;
2177
2178   pool_foreach (i, sm->interfaces, (
2179                                      {
2180                                      if (i->sw_if_index == sw_if_index)
2181                                      {
2182                                      if (!(nat_interface_is_outside (i)))
2183                                      return;}
2184                                      }
2185                 ));
2186   vec_foreach (outside_fib, sm->outside_fibs)
2187   {
2188     if (outside_fib->fib_index == old_fib_index)
2189       {
2190         outside_fib->refcount--;
2191         if (!outside_fib->refcount)
2192           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2193         break;
2194       }
2195   }
2196
2197   vec_foreach (outside_fib, sm->outside_fibs)
2198   {
2199     if (outside_fib->fib_index == new_fib_index)
2200       {
2201         outside_fib->refcount++;
2202         is_add = 0;
2203         break;
2204       }
2205   }
2206
2207   if (is_add)
2208     {
2209       vec_add2 (sm->outside_fibs, outside_fib, 1);
2210       outside_fib->refcount = 1;
2211       outside_fib->fib_index = new_fib_index;
2212     }
2213 }
2214
2215 static void
2216 snat_ip4_table_bind (ip4_main_t * im,
2217                      uword opaque,
2218                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2219 {
2220   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2221 }
2222
2223 static void
2224 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2225                                        uword opaque,
2226                                        u32 sw_if_index,
2227                                        ip4_address_t * address,
2228                                        u32 address_length,
2229                                        u32 if_address_index, u32 is_delete);
2230
2231 static void
2232 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2233                                  uword opaque,
2234                                  u32 sw_if_index,
2235                                  ip4_address_t * address,
2236                                  u32 address_length,
2237                                  u32 if_address_index, u32 is_delete);
2238
2239 static int
2240 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2241                                  u32 fib_index,
2242                                  u32 thread_index,
2243                                  snat_session_key_t * k,
2244                                  u16 port_per_thread, u32 snat_thread_index);
2245
2246 static clib_error_t *
2247 snat_init (vlib_main_t * vm)
2248 {
2249   snat_main_t *sm = &snat_main;
2250   clib_error_t *error = 0;
2251   ip4_main_t *im = &ip4_main;
2252   ip_lookup_main_t *lm = &im->lookup_main;
2253   uword *p;
2254   vlib_thread_registration_t *tr;
2255   vlib_thread_main_t *tm = vlib_get_thread_main ();
2256   uword *bitmap = 0;
2257   u32 i;
2258   ip4_add_del_interface_address_callback_t cb4;
2259   vlib_node_t *node;
2260
2261   sm->vlib_main = vm;
2262   sm->vnet_main = vnet_get_main ();
2263   sm->ip4_main = im;
2264   sm->ip4_lookup_main = lm;
2265   sm->api_main = &api_main;
2266   sm->first_worker_index = 0;
2267   sm->num_workers = 0;
2268   sm->num_snat_thread = 1;
2269   sm->workers = 0;
2270   sm->port_per_thread = 0xffff - 1024;
2271   sm->fq_in2out_index = ~0;
2272   sm->fq_out2in_index = ~0;
2273   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2274   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2275   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2276   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2277   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2278   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2279   sm->forwarding_enabled = 0;
2280   sm->log_class = vlib_log_register_class ("nat", 0);
2281   sm->mss_clamping = 0;
2282
2283   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2284   sm->error_node_index = node->index;
2285
2286   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2287   sm->in2out_node_index = node->index;
2288   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2289   sm->in2out_output_node_index = node->index;
2290   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2291   sm->in2out_fast_node_index = node->index;
2292   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2293   sm->in2out_slowpath_node_index = node->index;
2294   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2295   sm->in2out_slowpath_output_node_index = node->index;
2296   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-reass");
2297   sm->in2out_reass_node_index = node->index;
2298
2299   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2300   sm->ed_in2out_node_index = node->index;
2301   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2302   sm->ed_in2out_slowpath_node_index = node->index;
2303   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-reass");
2304   sm->ed_in2out_reass_node_index = node->index;
2305
2306   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2307   sm->out2in_node_index = node->index;
2308   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2309   sm->out2in_fast_node_index = node->index;
2310   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-reass");
2311   sm->out2in_reass_node_index = node->index;
2312
2313   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2314   sm->ed_out2in_node_index = node->index;
2315   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2316   sm->ed_out2in_slowpath_node_index = node->index;
2317   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-reass");
2318   sm->ed_out2in_reass_node_index = node->index;
2319
2320   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2321   sm->det_in2out_node_index = node->index;
2322   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2323   sm->det_out2in_node_index = node->index;
2324
2325   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2326   sm->hairpinning_node_index = node->index;
2327   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2328   sm->hairpin_dst_node_index = node->index;
2329   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2330   sm->hairpin_src_node_index = node->index;
2331   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2332   sm->ed_hairpinning_node_index = node->index;
2333   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2334   sm->ed_hairpin_dst_node_index = node->index;
2335   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2336   sm->ed_hairpin_src_node_index = node->index;
2337
2338   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2339   if (p)
2340     {
2341       tr = (vlib_thread_registration_t *) p[0];
2342       if (tr)
2343         {
2344           sm->num_workers = tr->count;
2345           sm->first_worker_index = tr->first_index;
2346         }
2347     }
2348
2349   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2350
2351   /* Use all available workers by default */
2352   if (sm->num_workers > 1)
2353     {
2354       for (i = 0; i < sm->num_workers; i++)
2355         bitmap = clib_bitmap_set (bitmap, i, 1);
2356       snat_set_workers (bitmap);
2357       clib_bitmap_free (bitmap);
2358     }
2359   else
2360     {
2361       sm->per_thread_data[0].snat_thread_index = 0;
2362     }
2363
2364   error = snat_api_init (vm, sm);
2365   if (error)
2366     return error;
2367
2368   /* Set up the interface address add/del callback */
2369   cb4.function = snat_ip4_add_del_interface_address_cb;
2370   cb4.function_opaque = 0;
2371
2372   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2373
2374   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2375   cb4.function_opaque = 0;
2376
2377   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2378
2379   nat_dpo_module_init ();
2380
2381   /* Init counters */
2382   sm->total_users.name = "total-users";
2383   sm->total_users.stat_segment_name = "/nat44/total-users";
2384   vlib_validate_simple_counter (&sm->total_users, 0);
2385   vlib_zero_simple_counter (&sm->total_users, 0);
2386   sm->total_sessions.name = "total-sessions";
2387   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2388   vlib_validate_simple_counter (&sm->total_sessions, 0);
2389   vlib_zero_simple_counter (&sm->total_sessions, 0);
2390
2391   /* Init IPFIX logging */
2392   snat_ipfix_logging_init (vm);
2393
2394   /* Init NAT64 */
2395   error = nat64_init (vm);
2396   if (error)
2397     return error;
2398
2399   dslite_init (vm);
2400
2401   nat66_init (vm);
2402
2403   ip4_table_bind_callback_t cbt4 = {
2404     .function = snat_ip4_table_bind,
2405   };
2406   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2407
2408   /* Init virtual fragmenentation reassembly */
2409   return nat_reass_init (vm);
2410 }
2411
2412 VLIB_INIT_FUNCTION (snat_init);
2413
2414 void
2415 snat_free_outside_address_and_port (snat_address_t * addresses,
2416                                     u32 thread_index, snat_session_key_t * k)
2417 {
2418   snat_address_t *a;
2419   u32 address_index;
2420   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2421
2422   for (address_index = 0; address_index < vec_len (addresses);
2423        address_index++)
2424     {
2425       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2426         break;
2427     }
2428
2429   ASSERT (address_index < vec_len (addresses));
2430
2431   a = addresses + address_index;
2432
2433   switch (k->protocol)
2434     {
2435 #define _(N, i, n, s) \
2436     case SNAT_PROTOCOL_##N: \
2437       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2438         port_host_byte_order) == 1); \
2439       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2440         port_host_byte_order, 0); \
2441       a->busy_##n##_ports--; \
2442       a->busy_##n##_ports_per_thread[thread_index]--; \
2443       break;
2444       foreach_snat_protocol
2445 #undef _
2446     default:
2447       nat_log_info ("unknown protocol");
2448       return;
2449     }
2450 }
2451
2452 static int
2453 nat_set_outside_address_and_port (snat_address_t * addresses,
2454                                   u32 thread_index, snat_session_key_t * k)
2455 {
2456   snat_address_t *a = 0;
2457   u32 address_index;
2458   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2459
2460   for (address_index = 0; address_index < vec_len (addresses);
2461        address_index++)
2462     {
2463       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2464         continue;
2465
2466       a = addresses + address_index;
2467       switch (k->protocol)
2468         {
2469 #define _(N, j, n, s) \
2470         case SNAT_PROTOCOL_##N: \
2471           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2472             return VNET_API_ERROR_INSTANCE_IN_USE; \
2473           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2474           a->busy_##n##_ports_per_thread[thread_index]++; \
2475           a->busy_##n##_ports++; \
2476           return 0;
2477           foreach_snat_protocol
2478 #undef _
2479         default:
2480           nat_log_info ("unknown protocol");
2481           return 1;
2482         }
2483     }
2484
2485   return VNET_API_ERROR_NO_SUCH_ENTRY;
2486 }
2487
2488 int
2489 snat_static_mapping_match (snat_main_t * sm,
2490                            snat_session_key_t match,
2491                            snat_session_key_t * mapping,
2492                            u8 by_external,
2493                            u8 * is_addr_only,
2494                            twice_nat_type_t * twice_nat,
2495                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2496                            u8 * is_identity_nat)
2497 {
2498   clib_bihash_kv_8_8_t kv, value;
2499   snat_static_mapping_t *m;
2500   snat_session_key_t m_key;
2501   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2502   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2503   u8 backend_index;
2504   nat44_lb_addr_port_t *local;
2505
2506   m_key.fib_index = match.fib_index;
2507   if (by_external)
2508     {
2509       mapping_hash = &sm->static_mapping_by_external;
2510       m_key.fib_index = 0;
2511     }
2512
2513   m_key.addr = match.addr;
2514   m_key.port = clib_net_to_host_u16 (match.port);
2515   m_key.protocol = match.protocol;
2516
2517   kv.key = m_key.as_u64;
2518
2519   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2520     {
2521       /* Try address only mapping */
2522       m_key.port = 0;
2523       m_key.protocol = 0;
2524       kv.key = m_key.as_u64;
2525       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2526         return 1;
2527     }
2528
2529   m = pool_elt_at_index (sm->static_mappings, value.value);
2530
2531   if (by_external)
2532     {
2533       if (is_lb_static_mapping (m))
2534         {
2535           if (PREDICT_FALSE (lb != 0))
2536             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2537           if (m->affinity)
2538             {
2539               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2540                                               match.protocol, match.port,
2541                                               &backend_index))
2542                 goto get_local;
2543
2544               local = pool_elt_at_index (m->locals, backend_index);
2545               mapping->addr = local->addr;
2546               mapping->port = clib_host_to_net_u16 (local->port);
2547               mapping->fib_index = local->fib_index;
2548               goto end;
2549             }
2550         get_local:
2551           /* *INDENT-OFF* */
2552           pool_foreach_index (i, m->locals,
2553           ({
2554             vec_add1 (tmp, i);
2555           }));
2556           /* *INDENT-ON* */
2557           hi = vec_len (tmp) - 1;
2558           local = pool_elt_at_index (m->locals, tmp[hi]);
2559           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2560           while (lo < hi)
2561             {
2562               mid = ((hi - lo) >> 1) + lo;
2563               local = pool_elt_at_index (m->locals, tmp[mid]);
2564               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2565             }
2566           local = pool_elt_at_index (m->locals, tmp[lo]);
2567           if (!(local->prefix >= rand))
2568             return 1;
2569           if (PREDICT_FALSE (sm->num_workers > 1))
2570             {
2571               ip4_header_t ip = {
2572                 .src_address = local->addr,
2573               };
2574               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2575                   vlib_get_thread_index ())
2576                 goto get_local;
2577             }
2578           mapping->addr = local->addr;
2579           mapping->port = clib_host_to_net_u16 (local->port);
2580           mapping->fib_index = local->fib_index;
2581           if (m->affinity)
2582             {
2583               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2584                                                 match.protocol, match.port,
2585                                                 tmp[lo], m->affinity,
2586                                                 m->affinity_per_service_list_head_index))
2587                 nat_log_info ("create affinity record failed");
2588             }
2589           vec_free (tmp);
2590         }
2591       else
2592         {
2593           if (PREDICT_FALSE (lb != 0))
2594             *lb = NO_LB_NAT;
2595           mapping->fib_index = m->fib_index;
2596           mapping->addr = m->local_addr;
2597           /* Address only mapping doesn't change port */
2598           mapping->port = is_addr_only_static_mapping (m) ? match.port
2599             : clib_host_to_net_u16 (m->local_port);
2600         }
2601       mapping->protocol = m->proto;
2602     }
2603   else
2604     {
2605       mapping->addr = m->external_addr;
2606       /* Address only mapping doesn't change port */
2607       mapping->port = is_addr_only_static_mapping (m) ? match.port
2608         : clib_host_to_net_u16 (m->external_port);
2609       mapping->fib_index = sm->outside_fib_index;
2610     }
2611
2612 end:
2613   if (PREDICT_FALSE (is_addr_only != 0))
2614     *is_addr_only = is_addr_only_static_mapping (m);
2615
2616   if (PREDICT_FALSE (twice_nat != 0))
2617     *twice_nat = m->twice_nat;
2618
2619   if (PREDICT_FALSE (is_identity_nat != 0))
2620     *is_identity_nat = is_identity_static_mapping (m);
2621
2622   return 0;
2623 }
2624
2625 static_always_inline u16
2626 snat_random_port (u16 min, u16 max)
2627 {
2628   snat_main_t *sm = &snat_main;
2629   return min + random_u32 (&sm->random_seed) /
2630     (random_u32_max () / (max - min + 1) + 1);
2631 }
2632
2633 int
2634 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2635                                      u32 fib_index,
2636                                      u32 thread_index,
2637                                      snat_session_key_t * k,
2638                                      u16 port_per_thread,
2639                                      u32 snat_thread_index)
2640 {
2641   snat_main_t *sm = &snat_main;
2642
2643   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2644                                   port_per_thread, snat_thread_index);
2645 }
2646
2647 static int
2648 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2649                                  u32 fib_index,
2650                                  u32 thread_index,
2651                                  snat_session_key_t * k,
2652                                  u16 port_per_thread, u32 snat_thread_index)
2653 {
2654   int i;
2655   snat_address_t *a, *ga = 0;
2656   u32 portnum;
2657
2658   for (i = 0; i < vec_len (addresses); i++)
2659     {
2660       a = addresses + i;
2661       switch (k->protocol)
2662         {
2663 #define _(N, j, n, s) \
2664         case SNAT_PROTOCOL_##N: \
2665           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2666             { \
2667               if (a->fib_index == fib_index) \
2668                 { \
2669                   while (1) \
2670                     { \
2671                       portnum = (port_per_thread * \
2672                         snat_thread_index) + \
2673                         snat_random_port(1, port_per_thread) + 1024; \
2674                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2675                         continue; \
2676                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2677                       a->busy_##n##_ports_per_thread[thread_index]++; \
2678                       a->busy_##n##_ports++; \
2679                       k->addr = a->addr; \
2680                       k->port = clib_host_to_net_u16(portnum); \
2681                       return 0; \
2682                     } \
2683                 } \
2684               else if (a->fib_index == ~0) \
2685                 { \
2686                   ga = a; \
2687                 } \
2688             } \
2689           break;
2690           foreach_snat_protocol
2691 #undef _
2692         default:
2693           nat_log_info ("unknown protocol");
2694           return 1;
2695         }
2696
2697     }
2698
2699   if (ga)
2700     {
2701       a = ga;
2702       switch (k->protocol)
2703         {
2704 #define _(N, j, n, s) \
2705         case SNAT_PROTOCOL_##N: \
2706           while (1) \
2707             { \
2708               portnum = (port_per_thread * \
2709                 snat_thread_index) + \
2710                 snat_random_port(1, port_per_thread) + 1024; \
2711               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2712                 continue; \
2713               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2714               a->busy_##n##_ports_per_thread[thread_index]++; \
2715               a->busy_##n##_ports++; \
2716               k->addr = a->addr; \
2717               k->port = clib_host_to_net_u16(portnum); \
2718               return 0; \
2719             }
2720           break;
2721           foreach_snat_protocol
2722 #undef _
2723         default:
2724           nat_log_info ("unknown protocol");
2725           return 1;
2726         }
2727     }
2728
2729   /* Totally out of translations to use... */
2730   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2731   return 1;
2732 }
2733
2734 static int
2735 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2736                               u32 fib_index,
2737                               u32 thread_index,
2738                               snat_session_key_t * k,
2739                               u16 port_per_thread, u32 snat_thread_index)
2740 {
2741   snat_main_t *sm = &snat_main;
2742   snat_address_t *a = addresses;
2743   u16 m, ports, portnum, A, j;
2744   m = 16 - (sm->psid_offset + sm->psid_length);
2745   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2746
2747   if (!vec_len (addresses))
2748     goto exhausted;
2749
2750   switch (k->protocol)
2751     {
2752 #define _(N, i, n, s) \
2753     case SNAT_PROTOCOL_##N: \
2754       if (a->busy_##n##_ports < ports) \
2755         { \
2756           while (1) \
2757             { \
2758               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2759               j = snat_random_port(0, pow2_mask(m)); \
2760               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2761               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2762                 continue; \
2763               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2764               a->busy_##n##_ports++; \
2765               k->addr = a->addr; \
2766               k->port = clib_host_to_net_u16 (portnum); \
2767               return 0; \
2768             } \
2769         } \
2770       break;
2771       foreach_snat_protocol
2772 #undef _
2773     default:
2774       nat_log_info ("unknown protocol");
2775       return 1;
2776     }
2777
2778 exhausted:
2779   /* Totally out of translations to use... */
2780   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2781   return 1;
2782 }
2783
2784 static int
2785 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2786                                u32 fib_index,
2787                                u32 thread_index,
2788                                snat_session_key_t * k,
2789                                u16 port_per_thread, u32 snat_thread_index)
2790 {
2791   snat_main_t *sm = &snat_main;
2792   snat_address_t *a = addresses;
2793   u16 portnum, ports;
2794
2795   ports = sm->end_port - sm->start_port + 1;
2796
2797   if (!vec_len (addresses))
2798     goto exhausted;
2799
2800   switch (k->protocol)
2801     {
2802 #define _(N, i, n, s) \
2803     case SNAT_PROTOCOL_##N: \
2804       if (a->busy_##n##_ports < ports) \
2805         { \
2806           while (1) \
2807             { \
2808               portnum = snat_random_port(sm->start_port, sm->end_port); \
2809               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2810                 continue; \
2811               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2812               a->busy_##n##_ports++; \
2813               k->addr = a->addr; \
2814               k->port = clib_host_to_net_u16 (portnum); \
2815               return 0; \
2816             } \
2817         } \
2818       break;
2819       foreach_snat_protocol
2820 #undef _
2821     default:
2822       nat_log_info ("unknown protocol");
2823       return 1;
2824     }
2825
2826 exhausted:
2827   /* Totally out of translations to use... */
2828   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2829   return 1;
2830 }
2831
2832 void
2833 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2834 {
2835   dpo_id_t dpo_v4 = DPO_INVALID;
2836   fib_prefix_t pfx = {
2837     .fp_proto = FIB_PROTOCOL_IP4,
2838     .fp_len = 32,
2839     .fp_addr.ip4.as_u32 = addr.as_u32,
2840   };
2841
2842   if (is_add)
2843     {
2844       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2845       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2846                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2847       dpo_reset (&dpo_v4);
2848     }
2849   else
2850     {
2851       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2852     }
2853 }
2854
2855 u8 *
2856 format_session_kvp (u8 * s, va_list * args)
2857 {
2858   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2859   snat_session_key_t k;
2860
2861   k.as_u64 = v->key;
2862
2863   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2864
2865   return s;
2866 }
2867
2868 u8 *
2869 format_static_mapping_kvp (u8 * s, va_list * args)
2870 {
2871   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2872   snat_session_key_t k;
2873
2874   k.as_u64 = v->key;
2875
2876   s = format (s, "%U static-mapping-index %llu",
2877               format_static_mapping_key, &k, v->value);
2878
2879   return s;
2880 }
2881
2882 u8 *
2883 format_user_kvp (u8 * s, va_list * args)
2884 {
2885   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2886   snat_user_key_t k;
2887
2888   k.as_u64 = v->key;
2889
2890   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2891               k.fib_index, v->value);
2892
2893   return s;
2894 }
2895
2896 u8 *
2897 format_ed_session_kvp (u8 * s, va_list * args)
2898 {
2899   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2900   nat_ed_ses_key_t k;
2901
2902   k.as_u64[0] = v->key[0];
2903   k.as_u64[1] = v->key[1];
2904
2905   s =
2906     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2907             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2908             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2909             format_ip_protocol, k.proto, k.fib_index, v->value);
2910
2911   return s;
2912 }
2913
2914 static u32
2915 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2916 {
2917   snat_main_t *sm = &snat_main;
2918   u32 next_worker_index = 0;
2919   u32 hash;
2920
2921   next_worker_index = sm->first_worker_index;
2922   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2923     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2924
2925   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2926     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2927   else
2928     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2929
2930   return next_worker_index;
2931 }
2932
2933 static u32
2934 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2935 {
2936   snat_main_t *sm = &snat_main;
2937   udp_header_t *udp;
2938   u16 port;
2939   snat_session_key_t m_key;
2940   clib_bihash_kv_8_8_t kv, value;
2941   snat_static_mapping_t *m;
2942   u32 proto;
2943   u32 next_worker_index = 0;
2944
2945   /* first try static mappings without port */
2946   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2947     {
2948       m_key.addr = ip0->dst_address;
2949       m_key.port = 0;
2950       m_key.protocol = 0;
2951       m_key.fib_index = rx_fib_index0;
2952       kv.key = m_key.as_u64;
2953       if (!clib_bihash_search_8_8
2954           (&sm->static_mapping_by_external, &kv, &value))
2955         {
2956           m = pool_elt_at_index (sm->static_mappings, value.value);
2957           return m->workers[0];
2958         }
2959     }
2960
2961   proto = ip_proto_to_snat_proto (ip0->protocol);
2962   udp = ip4_next_header (ip0);
2963   port = udp->dst_port;
2964
2965   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2966     {
2967       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2968         return vlib_get_thread_index ();
2969
2970       nat_reass_ip4_t *reass;
2971       reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2972                                   ip0->fragment_id, ip0->protocol);
2973
2974       if (reass && (reass->thread_index != (u32) ~ 0))
2975         return reass->thread_index;
2976
2977       if (ip4_is_first_fragment (ip0))
2978         {
2979           reass =
2980             nat_ip4_reass_create (ip0->src_address, ip0->dst_address,
2981                                   ip0->fragment_id, ip0->protocol);
2982           if (!reass)
2983             goto no_reass;
2984
2985           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2986             {
2987               m_key.addr = ip0->dst_address;
2988               m_key.port = clib_net_to_host_u16 (port);
2989               m_key.protocol = proto;
2990               m_key.fib_index = rx_fib_index0;
2991               kv.key = m_key.as_u64;
2992               if (!clib_bihash_search_8_8
2993                   (&sm->static_mapping_by_external, &kv, &value))
2994                 {
2995                   m = pool_elt_at_index (sm->static_mappings, value.value);
2996                   reass->thread_index = m->workers[0];
2997                   return reass->thread_index;
2998                 }
2999             }
3000           reass->thread_index = sm->first_worker_index;
3001           reass->thread_index +=
3002             sm->workers[(clib_net_to_host_u16 (port) - 1024) /
3003                         sm->port_per_thread];
3004           return reass->thread_index;
3005         }
3006       else
3007         return vlib_get_thread_index ();
3008     }
3009
3010 no_reass:
3011   /* unknown protocol */
3012   if (PREDICT_FALSE (proto == ~0))
3013     {
3014       /* use current thread */
3015       return vlib_get_thread_index ();
3016     }
3017
3018   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3019     {
3020       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3021       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3022       if (!icmp_is_error_message (icmp))
3023         port = echo->identifier;
3024       else
3025         {
3026           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3027           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3028           void *l4_header = ip4_next_header (inner_ip);
3029           switch (proto)
3030             {
3031             case SNAT_PROTOCOL_ICMP:
3032               icmp = (icmp46_header_t *) l4_header;
3033               echo = (icmp_echo_header_t *) (icmp + 1);
3034               port = echo->identifier;
3035               break;
3036             case SNAT_PROTOCOL_UDP:
3037             case SNAT_PROTOCOL_TCP:
3038               port = ((tcp_udp_header_t *) l4_header)->src_port;
3039               break;
3040             default:
3041               return vlib_get_thread_index ();
3042             }
3043         }
3044     }
3045
3046   /* try static mappings with port */
3047   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3048     {
3049       m_key.addr = ip0->dst_address;
3050       m_key.port = clib_net_to_host_u16 (port);
3051       m_key.protocol = proto;
3052       m_key.fib_index = rx_fib_index0;
3053       kv.key = m_key.as_u64;
3054       if (!clib_bihash_search_8_8
3055           (&sm->static_mapping_by_external, &kv, &value))
3056         {
3057           m = pool_elt_at_index (sm->static_mappings, value.value);
3058           return m->workers[0];
3059         }
3060     }
3061
3062   /* worker by outside port */
3063   next_worker_index = sm->first_worker_index;
3064   next_worker_index +=
3065     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3066   return next_worker_index;
3067 }
3068
3069 static u32
3070 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
3071 {
3072   snat_main_t *sm = &snat_main;
3073   clib_bihash_kv_8_8_t kv, value;
3074   u32 proto, next_worker_index = 0;
3075   udp_header_t *udp;
3076   u16 port;
3077   snat_static_mapping_t *m;
3078   u32 hash;
3079
3080   /* first try static mappings without port */
3081   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3082     {
3083       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
3084       if (!clib_bihash_search_8_8
3085           (&sm->static_mapping_by_external, &kv, &value))
3086         {
3087           m = pool_elt_at_index (sm->static_mappings, value.value);
3088           return m->workers[0];
3089         }
3090     }
3091
3092   proto = ip_proto_to_snat_proto (ip->protocol);
3093
3094   /* unknown protocol */
3095   if (PREDICT_FALSE (proto == ~0))
3096     {
3097       /* use current thread */
3098       return vlib_get_thread_index ();
3099     }
3100
3101   udp = ip4_next_header (ip);
3102   port = udp->dst_port;
3103
3104   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3105     {
3106       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3107       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3108       if (!icmp_is_error_message (icmp))
3109         port = echo->identifier;
3110       else
3111         {
3112           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3113           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3114           void *l4_header = ip4_next_header (inner_ip);
3115           switch (proto)
3116             {
3117             case SNAT_PROTOCOL_ICMP:
3118               icmp = (icmp46_header_t *) l4_header;
3119               echo = (icmp_echo_header_t *) (icmp + 1);
3120               port = echo->identifier;
3121               break;
3122             case SNAT_PROTOCOL_UDP:
3123             case SNAT_PROTOCOL_TCP:
3124               port = ((tcp_udp_header_t *) l4_header)->src_port;
3125               break;
3126             default:
3127               return vlib_get_thread_index ();
3128             }
3129         }
3130     }
3131
3132   /* try static mappings with port */
3133   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3134     {
3135       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
3136                   clib_net_to_host_u16 (port));
3137       if (!clib_bihash_search_8_8
3138           (&sm->static_mapping_by_external, &kv, &value))
3139         {
3140           m = pool_elt_at_index (sm->static_mappings, value.value);
3141           if (!is_lb_static_mapping (m))
3142             return m->workers[0];
3143
3144           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3145             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3146
3147           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3148             return m->workers[hash & (_vec_len (m->workers) - 1)];
3149           else
3150             return m->workers[hash % _vec_len (m->workers)];
3151         }
3152     }
3153
3154   /* worker by outside port */
3155   next_worker_index = sm->first_worker_index;
3156   next_worker_index +=
3157     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3158
3159   return next_worker_index;
3160 }
3161
3162 void
3163 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3164                 ip4_address_t * out_addr, u16 out_port,
3165                 ip4_address_t * eh_addr, u16 eh_port,
3166                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3167                 u32 fib_index, u16 flags, u32 thread_index)
3168 {
3169   snat_main_t *sm = &snat_main;
3170   snat_session_key_t key;
3171   snat_user_t *u;
3172   snat_session_t *s;
3173   clib_bihash_kv_8_8_t kv;
3174   f64 now = vlib_time_now (sm->vlib_main);
3175   nat_outside_fib_t *outside_fib;
3176   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3177   snat_main_per_thread_data_t *tsm;
3178   fib_prefix_t pfx = {
3179     .fp_proto = FIB_PROTOCOL_IP4,
3180     .fp_len = 32,
3181     .fp_addr = {
3182                 .ip4.as_u32 = eh_addr->as_u32,
3183                 },
3184   };
3185
3186   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3187
3188   key.addr.as_u32 = out_addr->as_u32;
3189   key.port = out_port;
3190   key.protocol = proto;
3191
3192   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3193     {
3194       if (nat_set_outside_address_and_port
3195           (sm->addresses, thread_index, &key))
3196         return;
3197     }
3198
3199   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3200   if (!u)
3201     return;
3202
3203   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3204   if (!s)
3205     return;
3206
3207   s->last_heard = now;
3208   s->flags = flags;
3209   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3210   s->ext_host_port = eh_port;
3211   user_session_increment (sm, u, snat_is_session_static (s));
3212   switch (vec_len (sm->outside_fibs))
3213     {
3214     case 0:
3215       key.fib_index = sm->outside_fib_index;
3216       break;
3217     case 1:
3218       key.fib_index = sm->outside_fibs[0].fib_index;
3219       break;
3220     default:
3221       /* *INDENT-OFF* */
3222       vec_foreach (outside_fib, sm->outside_fibs)
3223         {
3224           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3225           if (FIB_NODE_INDEX_INVALID != fei)
3226             {
3227               if (fib_entry_get_resolving_interface (fei) != ~0)
3228                 {
3229                   key.fib_index = outside_fib->fib_index;
3230                   break;
3231                 }
3232             }
3233         }
3234       /* *INDENT-ON* */
3235       break;
3236     }
3237   s->out2in = key;
3238   kv.key = key.as_u64;
3239   kv.value = s - tsm->sessions;
3240   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3241     nat_log_warn ("out2in key add failed");
3242
3243   key.addr.as_u32 = in_addr->as_u32;
3244   key.port = in_port;
3245   key.fib_index = fib_index;
3246   s->in2out = key;
3247   kv.key = key.as_u64;
3248   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3249     nat_log_warn ("in2out key add failed");
3250 }
3251
3252 void
3253 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3254                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3255                 u32 ti)
3256 {
3257   snat_main_t *sm = &snat_main;
3258   snat_session_key_t key;
3259   clib_bihash_kv_8_8_t kv, value;
3260   u32 thread_index;
3261   snat_session_t *s;
3262   snat_main_per_thread_data_t *tsm;
3263
3264   if (sm->num_workers > 1)
3265     thread_index =
3266       sm->first_worker_index +
3267       (sm->workers[(clib_net_to_host_u16 (out_port) -
3268                     1024) / sm->port_per_thread]);
3269   else
3270     thread_index = sm->num_workers;
3271   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3272
3273   key.addr.as_u32 = out_addr->as_u32;
3274   key.port = out_port;
3275   key.protocol = proto;
3276   key.fib_index = fib_index;
3277   kv.key = key.as_u64;
3278   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3279     return;
3280
3281   s = pool_elt_at_index (tsm->sessions, value.value);
3282   nat_free_session_data (sm, s, thread_index, 1);
3283   nat44_delete_session (sm, s, thread_index);
3284 }
3285
3286 void
3287 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3288                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3289                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3290 {
3291   snat_main_t *sm = &snat_main;
3292   snat_session_key_t key;
3293   clib_bihash_kv_8_8_t kv, value;
3294   snat_session_t *s;
3295   snat_main_per_thread_data_t *tsm;
3296
3297   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3298
3299   key.addr.as_u32 = out_addr->as_u32;
3300   key.port = out_port;
3301   key.protocol = proto;
3302   key.fib_index = fib_index;
3303   kv.key = key.as_u64;
3304   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3305     return;
3306
3307   s = pool_elt_at_index (tsm->sessions, value.value);
3308   s->total_pkts = total_pkts;
3309   s->total_bytes = total_bytes;
3310 }
3311
3312 void
3313 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3314                    ip4_address_t * out_addr, u16 out_port,
3315                    ip4_address_t * eh_addr, u16 eh_port,
3316                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3317                    u32 fib_index, u16 flags, u32 thread_index)
3318 {
3319   snat_main_t *sm = &snat_main;
3320   snat_session_key_t key;
3321   snat_user_t *u;
3322   snat_session_t *s;
3323   clib_bihash_kv_16_8_t kv;
3324   f64 now = vlib_time_now (sm->vlib_main);
3325   nat_outside_fib_t *outside_fib;
3326   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3327   snat_main_per_thread_data_t *tsm;
3328   fib_prefix_t pfx = {
3329     .fp_proto = FIB_PROTOCOL_IP4,
3330     .fp_len = 32,
3331     .fp_addr = {
3332                 .ip4.as_u32 = eh_addr->as_u32,
3333                 },
3334   };
3335
3336   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3337
3338   key.addr.as_u32 = out_addr->as_u32;
3339   key.port = out_port;
3340   key.protocol = proto;
3341
3342   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3343     {
3344       if (nat_set_outside_address_and_port
3345           (sm->addresses, thread_index, &key))
3346         return;
3347     }
3348
3349   key.addr.as_u32 = ehn_addr->as_u32;
3350   key.port = ehn_port;
3351   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3352     {
3353       if (nat_set_outside_address_and_port
3354           (sm->twice_nat_addresses, thread_index, &key))
3355         return;
3356     }
3357
3358   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3359   if (!u)
3360     return;
3361
3362   s = nat_ed_session_alloc (sm, u, thread_index, now);
3363   if (!s)
3364     return;
3365
3366   s->last_heard = now;
3367   s->flags = flags;
3368   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3369   s->ext_host_nat_port = s->ext_host_port = eh_port;
3370   if (is_twice_nat_session (s))
3371     {
3372       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3373       s->ext_host_nat_port = ehn_port;
3374     }
3375   user_session_increment (sm, u, snat_is_session_static (s));
3376   switch (vec_len (sm->outside_fibs))
3377     {
3378     case 0:
3379       key.fib_index = sm->outside_fib_index;
3380       break;
3381     case 1:
3382       key.fib_index = sm->outside_fibs[0].fib_index;
3383       break;
3384     default:
3385       /* *INDENT-OFF* */
3386       vec_foreach (outside_fib, sm->outside_fibs)
3387         {
3388           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3389           if (FIB_NODE_INDEX_INVALID != fei)
3390             {
3391               if (fib_entry_get_resolving_interface (fei) != ~0)
3392                 {
3393                   key.fib_index = outside_fib->fib_index;
3394                   break;
3395                 }
3396             }
3397         }
3398       /* *INDENT-ON* */
3399       break;
3400     }
3401   key.addr.as_u32 = out_addr->as_u32;
3402   key.port = out_port;
3403   s->out2in = key;
3404   kv.value = s - tsm->sessions;
3405
3406   key.addr.as_u32 = in_addr->as_u32;
3407   key.port = in_port;
3408   key.fib_index = fib_index;
3409   s->in2out = key;
3410
3411   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3412               snat_proto_to_ip_proto (proto), fib_index, in_port,
3413               s->ext_host_nat_port);
3414   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3415     nat_log_warn ("in2out key add failed");
3416
3417   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3418               s->out2in.fib_index, out_port, eh_port);
3419   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3420     nat_log_warn ("out2in key add failed");
3421 }
3422
3423 void
3424 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3425                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3426                    u32 fib_index, u32 ti)
3427 {
3428   snat_main_t *sm = &snat_main;
3429   nat_ed_ses_key_t key;
3430   clib_bihash_kv_16_8_t kv, value;
3431   u32 thread_index;
3432   snat_session_t *s;
3433   snat_main_per_thread_data_t *tsm;
3434
3435   if (sm->num_workers > 1)
3436     thread_index =
3437       sm->first_worker_index +
3438       (sm->workers[(clib_net_to_host_u16 (out_port) -
3439                     1024) / sm->port_per_thread]);
3440   else
3441     thread_index = sm->num_workers;
3442   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3443
3444   key.l_addr.as_u32 = out_addr->as_u32;
3445   key.l_port = out_port;
3446   key.r_addr.as_u32 = eh_addr->as_u32;
3447   key.r_port = eh_port;
3448   key.proto = proto;
3449   key.fib_index = fib_index;
3450   kv.key[0] = key.as_u64[0];
3451   kv.key[1] = key.as_u64[1];
3452   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3453     return;
3454
3455   s = pool_elt_at_index (tsm->sessions, value.value);
3456   nat_free_session_data (sm, s, thread_index, 1);
3457   nat44_delete_session (sm, s, thread_index);
3458 }
3459
3460 void
3461 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3462                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3463                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3464                    u32 thread_index)
3465 {
3466   snat_main_t *sm = &snat_main;
3467   nat_ed_ses_key_t key;
3468   clib_bihash_kv_16_8_t kv, value;
3469   snat_session_t *s;
3470   snat_main_per_thread_data_t *tsm;
3471
3472   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3473
3474   key.l_addr.as_u32 = out_addr->as_u32;
3475   key.l_port = out_port;
3476   key.r_addr.as_u32 = eh_addr->as_u32;
3477   key.r_port = eh_port;
3478   key.proto = proto;
3479   key.fib_index = fib_index;
3480   kv.key[0] = key.as_u64[0];
3481   kv.key[1] = key.as_u64[1];
3482   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3483     return;
3484
3485   s = pool_elt_at_index (tsm->sessions, value.value);
3486   s->total_pkts = total_pkts;
3487   s->total_bytes = total_bytes;
3488 }
3489
3490 static clib_error_t *
3491 snat_config (vlib_main_t * vm, unformat_input_t * input)
3492 {
3493   snat_main_t *sm = &snat_main;
3494   nat66_main_t *nm = &nat66_main;
3495   u32 translation_buckets = 1024;
3496   u32 translation_memory_size = 128 << 20;
3497   u32 user_buckets = 128;
3498   u32 user_memory_size = 64 << 20;
3499   u32 max_translations_per_user = 100;
3500   u32 outside_vrf_id = 0;
3501   u32 outside_ip6_vrf_id = 0;
3502   u32 inside_vrf_id = 0;
3503   u32 static_mapping_buckets = 1024;
3504   u32 static_mapping_memory_size = 64 << 20;
3505   u32 nat64_bib_buckets = 1024;
3506   u32 nat64_bib_memory_size = 128 << 20;
3507   u32 nat64_st_buckets = 2048;
3508   u32 nat64_st_memory_size = 256 << 20;
3509   u8 static_mapping_only = 0;
3510   u8 static_mapping_connection_tracking = 0;
3511   snat_main_per_thread_data_t *tsm;
3512   dslite_main_t *dm = &dslite_main;
3513
3514   sm->deterministic = 0;
3515   sm->out2in_dpo = 0;
3516   sm->endpoint_dependent = 0;
3517
3518   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3519     {
3520       if (unformat
3521           (input, "translation hash buckets %d", &translation_buckets))
3522         ;
3523       else if (unformat (input, "translation hash memory %d",
3524                          &translation_memory_size));
3525       else if (unformat (input, "user hash buckets %d", &user_buckets))
3526         ;
3527       else if (unformat (input, "user hash memory %d", &user_memory_size))
3528         ;
3529       else if (unformat (input, "max translations per user %d",
3530                          &max_translations_per_user))
3531         ;
3532       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3533         ;
3534       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3535         ;
3536       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3537         ;
3538       else if (unformat (input, "static mapping only"))
3539         {
3540           static_mapping_only = 1;
3541           if (unformat (input, "connection tracking"))
3542             static_mapping_connection_tracking = 1;
3543         }
3544       else if (unformat (input, "deterministic"))
3545         sm->deterministic = 1;
3546       else if (unformat (input, "nat64 bib hash buckets %d",
3547                          &nat64_bib_buckets))
3548         ;
3549       else if (unformat (input, "nat64 bib hash memory %d",
3550                          &nat64_bib_memory_size))
3551         ;
3552       else
3553         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3554         ;
3555       else if (unformat (input, "nat64 st hash memory %d",
3556                          &nat64_st_memory_size))
3557         ;
3558       else if (unformat (input, "out2in dpo"))
3559         sm->out2in_dpo = 1;
3560       else if (unformat (input, "dslite ce"))
3561         dslite_set_ce (dm, 1);
3562       else if (unformat (input, "endpoint-dependent"))
3563         sm->endpoint_dependent = 1;
3564       else
3565         return clib_error_return (0, "unknown input '%U'",
3566                                   format_unformat_error, input);
3567     }
3568
3569   if (sm->deterministic && sm->endpoint_dependent)
3570     return clib_error_return (0,
3571                               "deterministic and endpoint-dependent modes are mutually exclusive");
3572
3573   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3574     return clib_error_return (0,
3575                               "static mapping only mode available only for simple nat");
3576
3577   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3578     return clib_error_return (0,
3579                               "out2in dpo mode available only for simple nat");
3580
3581   /* for show commands, etc. */
3582   sm->translation_buckets = translation_buckets;
3583   sm->translation_memory_size = translation_memory_size;
3584   /* do not exceed load factor 10 */
3585   sm->max_translations = 10 * translation_buckets;
3586   sm->user_buckets = user_buckets;
3587   sm->user_memory_size = user_memory_size;
3588   sm->max_translations_per_user = max_translations_per_user;
3589   sm->outside_vrf_id = outside_vrf_id;
3590   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3591                                                              outside_vrf_id,
3592                                                              FIB_SOURCE_PLUGIN_HI);
3593   nm->outside_vrf_id = outside_ip6_vrf_id;
3594   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3595                                                              outside_ip6_vrf_id,
3596                                                              FIB_SOURCE_PLUGIN_HI);
3597   sm->inside_vrf_id = inside_vrf_id;
3598   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3599                                                             inside_vrf_id,
3600                                                             FIB_SOURCE_PLUGIN_HI);
3601   sm->static_mapping_only = static_mapping_only;
3602   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3603
3604   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3605                   nat64_st_memory_size);
3606
3607   if (sm->deterministic)
3608     {
3609       sm->in2out_node_index = snat_det_in2out_node.index;
3610       sm->in2out_output_node_index = ~0;
3611       sm->out2in_node_index = snat_det_out2in_node.index;
3612       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3613       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3614     }
3615   else
3616     {
3617       if (sm->endpoint_dependent)
3618         {
3619           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3620           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3621           sm->in2out_node_index = nat44_ed_in2out_node.index;
3622           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3623           sm->out2in_node_index = nat44_ed_out2in_node.index;
3624           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3625           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3626           nat_affinity_init (vm);
3627           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
3628                        nat_ha_sref_ed_cb);
3629         }
3630       else
3631         {
3632           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3633           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3634           sm->in2out_node_index = snat_in2out_node.index;
3635           sm->in2out_output_node_index = snat_in2out_output_node.index;
3636           sm->out2in_node_index = snat_out2in_node.index;
3637           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3638           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3639           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
3640         }
3641       if (!static_mapping_only ||
3642           (static_mapping_only && static_mapping_connection_tracking))
3643         {
3644           /* *INDENT-OFF* */
3645           vec_foreach (tsm, sm->per_thread_data)
3646             {
3647               if (sm->endpoint_dependent)
3648                 {
3649                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3650                                          translation_buckets,
3651                                          translation_memory_size);
3652                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3653                                                       format_ed_session_kvp);
3654
3655                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3656                                          translation_buckets,
3657                                          translation_memory_size);
3658                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3659                                                       format_ed_session_kvp);
3660                 }
3661               else
3662                 {
3663                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3664                                         translation_buckets,
3665                                         translation_memory_size);
3666                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3667                                                      format_session_kvp);
3668
3669                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3670                                         translation_buckets,
3671                                         translation_memory_size);
3672                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3673                                                      format_session_kvp);
3674                 }
3675
3676               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3677                                     user_memory_size);
3678               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3679                                                  format_user_kvp);
3680             }
3681           /* *INDENT-ON* */
3682
3683         }
3684       else
3685         {
3686           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3687           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3688         }
3689       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3690                             "static_mapping_by_local", static_mapping_buckets,
3691                             static_mapping_memory_size);
3692       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3693                                          format_static_mapping_kvp);
3694
3695       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3696                             "static_mapping_by_external",
3697                             static_mapping_buckets,
3698                             static_mapping_memory_size);
3699       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3700                                          format_static_mapping_kvp);
3701     }
3702
3703   return 0;
3704 }
3705
3706 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3707
3708 static void
3709 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3710                                  uword opaque,
3711                                  u32 sw_if_index,
3712                                  ip4_address_t * address,
3713                                  u32 address_length,
3714                                  u32 if_address_index, u32 is_delete)
3715 {
3716   snat_main_t *sm = &snat_main;
3717   snat_static_map_resolve_t *rp;
3718   snat_static_mapping_t *m;
3719   snat_session_key_t m_key;
3720   clib_bihash_kv_8_8_t kv, value;
3721   int i, rv;
3722   ip4_address_t l_addr;
3723
3724   for (i = 0; i < vec_len (sm->to_resolve); i++)
3725     {
3726       rp = sm->to_resolve + i;
3727       if (rp->addr_only == 0)
3728         continue;
3729       if (rp->sw_if_index == sw_if_index)
3730         goto match;
3731     }
3732
3733   return;
3734
3735 match:
3736   m_key.addr.as_u32 = address->as_u32;
3737   m_key.port = rp->addr_only ? 0 : rp->e_port;
3738   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3739   m_key.fib_index = sm->outside_fib_index;
3740   kv.key = m_key.as_u64;
3741   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3742     m = 0;
3743   else
3744     m = pool_elt_at_index (sm->static_mappings, value.value);
3745
3746   if (!is_delete)
3747     {
3748       /* Don't trip over lease renewal, static config */
3749       if (m)
3750         return;
3751     }
3752   else
3753     {
3754       if (!m)
3755         return;
3756     }
3757
3758   /* Indetity mapping? */
3759   if (rp->l_addr.as_u32 == 0)
3760     l_addr.as_u32 = address[0].as_u32;
3761   else
3762     l_addr.as_u32 = rp->l_addr.as_u32;
3763   /* Add the static mapping */
3764   rv = snat_add_static_mapping (l_addr,
3765                                 address[0],
3766                                 rp->l_port,
3767                                 rp->e_port,
3768                                 rp->vrf_id,
3769                                 rp->addr_only, ~0 /* sw_if_index */ ,
3770                                 rp->proto, !is_delete, rp->twice_nat,
3771                                 rp->out2in_only, rp->tag, rp->identity_nat);
3772   if (rv)
3773     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3774 }
3775
3776 static void
3777 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3778                                        uword opaque,
3779                                        u32 sw_if_index,
3780                                        ip4_address_t * address,
3781                                        u32 address_length,
3782                                        u32 if_address_index, u32 is_delete)
3783 {
3784   snat_main_t *sm = &snat_main;
3785   snat_static_map_resolve_t *rp;
3786   ip4_address_t l_addr;
3787   int i, j;
3788   int rv;
3789   u8 twice_nat = 0;
3790   snat_address_t *addresses = sm->addresses;
3791
3792   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3793     {
3794       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3795         goto match;
3796     }
3797
3798   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3799     {
3800       twice_nat = 1;
3801       addresses = sm->twice_nat_addresses;
3802       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3803         goto match;
3804     }
3805
3806   return;
3807
3808 match:
3809   if (!is_delete)
3810     {
3811       /* Don't trip over lease renewal, static config */
3812       for (j = 0; j < vec_len (addresses); j++)
3813         if (addresses[j].addr.as_u32 == address->as_u32)
3814           return;
3815
3816       (void) snat_add_address (sm, address, ~0, twice_nat);
3817       /* Scan static map resolution vector */
3818       for (j = 0; j < vec_len (sm->to_resolve); j++)
3819         {
3820           rp = sm->to_resolve + j;
3821           if (rp->addr_only)
3822             continue;
3823           /* On this interface? */
3824           if (rp->sw_if_index == sw_if_index)
3825             {
3826               /* Indetity mapping? */
3827               if (rp->l_addr.as_u32 == 0)
3828                 l_addr.as_u32 = address[0].as_u32;
3829               else
3830                 l_addr.as_u32 = rp->l_addr.as_u32;
3831               /* Add the static mapping */
3832               rv = snat_add_static_mapping (l_addr,
3833                                             address[0],
3834                                             rp->l_port,
3835                                             rp->e_port,
3836                                             rp->vrf_id,
3837                                             rp->addr_only,
3838                                             ~0 /* sw_if_index */ ,
3839                                             rp->proto,
3840                                             rp->is_add, rp->twice_nat,
3841                                             rp->out2in_only, rp->tag,
3842                                             rp->identity_nat);
3843               if (rv)
3844                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3845             }
3846         }
3847       return;
3848     }
3849   else
3850     {
3851       (void) snat_del_address (sm, address[0], 1, twice_nat);
3852       return;
3853     }
3854 }
3855
3856
3857 int
3858 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3859                             u8 twice_nat)
3860 {
3861   ip4_main_t *ip4_main = sm->ip4_main;
3862   ip4_address_t *first_int_addr;
3863   snat_static_map_resolve_t *rp;
3864   u32 *indices_to_delete = 0;
3865   int i, j;
3866   u32 *auto_add_sw_if_indices =
3867     twice_nat ? sm->
3868     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3869
3870   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3871     );
3872
3873   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3874     {
3875       if (auto_add_sw_if_indices[i] == sw_if_index)
3876         {
3877           if (is_del)
3878             {
3879               /* if have address remove it */
3880               if (first_int_addr)
3881                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3882               else
3883                 {
3884                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3885                     {
3886                       rp = sm->to_resolve + j;
3887                       if (rp->sw_if_index == sw_if_index)
3888                         vec_add1 (indices_to_delete, j);
3889                     }
3890                   if (vec_len (indices_to_delete))
3891                     {
3892                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3893                         vec_del1 (sm->to_resolve, j);
3894                       vec_free (indices_to_delete);
3895                     }
3896                 }
3897               if (twice_nat)
3898                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3899               else
3900                 vec_del1 (sm->auto_add_sw_if_indices, i);
3901             }
3902           else
3903             return VNET_API_ERROR_VALUE_EXIST;
3904
3905           return 0;
3906         }
3907     }
3908
3909   if (is_del)
3910     return VNET_API_ERROR_NO_SUCH_ENTRY;
3911
3912   /* add to the auto-address list */
3913   if (twice_nat)
3914     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3915   else
3916     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3917
3918   /* If the address is already bound - or static - add it now */
3919   if (first_int_addr)
3920     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3921
3922   return 0;
3923 }
3924
3925 int
3926 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3927                    snat_protocol_t proto, u32 vrf_id, int is_in)
3928 {
3929   snat_main_per_thread_data_t *tsm;
3930   clib_bihash_kv_8_8_t kv, value;
3931   ip4_header_t ip;
3932   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3933   snat_session_key_t key;
3934   snat_session_t *s;
3935   clib_bihash_8_8_t *t;
3936
3937   if (sm->endpoint_dependent)
3938     return VNET_API_ERROR_UNSUPPORTED;
3939
3940   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3941   if (sm->num_workers > 1)
3942     tsm =
3943       vec_elt_at_index (sm->per_thread_data,
3944                         sm->worker_in2out_cb (&ip, fib_index));
3945   else
3946     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3947
3948   key.addr.as_u32 = addr->as_u32;
3949   key.port = clib_host_to_net_u16 (port);
3950   key.protocol = proto;
3951   key.fib_index = fib_index;
3952   kv.key = key.as_u64;
3953   t = is_in ? &tsm->in2out : &tsm->out2in;
3954   if (!clib_bihash_search_8_8 (t, &kv, &value))
3955     {
3956       if (pool_is_free_index (tsm->sessions, value.value))
3957         return VNET_API_ERROR_UNSPECIFIED;
3958
3959       s = pool_elt_at_index (tsm->sessions, value.value);
3960       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3961       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3962       return 0;
3963     }
3964
3965   return VNET_API_ERROR_NO_SUCH_ENTRY;
3966 }
3967
3968 int
3969 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3970                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3971                       u32 vrf_id, int is_in)
3972 {
3973   ip4_header_t ip;
3974   clib_bihash_16_8_t *t;
3975   nat_ed_ses_key_t key;
3976   clib_bihash_kv_16_8_t kv, value;
3977   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3978   snat_session_t *s;
3979   snat_main_per_thread_data_t *tsm;
3980
3981   if (!sm->endpoint_dependent)
3982     return VNET_API_ERROR_FEATURE_DISABLED;
3983
3984   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3985   if (sm->num_workers > 1)
3986     tsm =
3987       vec_elt_at_index (sm->per_thread_data,
3988                         sm->worker_in2out_cb (&ip, fib_index));
3989   else
3990     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3991
3992   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3993   key.l_addr.as_u32 = addr->as_u32;
3994   key.r_addr.as_u32 = eh_addr->as_u32;
3995   key.l_port = clib_host_to_net_u16 (port);
3996   key.r_port = clib_host_to_net_u16 (eh_port);
3997   key.proto = proto;
3998   key.fib_index = fib_index;
3999   kv.key[0] = key.as_u64[0];
4000   kv.key[1] = key.as_u64[1];
4001   if (clib_bihash_search_16_8 (t, &kv, &value))
4002     return VNET_API_ERROR_NO_SUCH_ENTRY;
4003
4004   if (pool_is_free_index (tsm->sessions, value.value))
4005     return VNET_API_ERROR_UNSPECIFIED;
4006   s = pool_elt_at_index (tsm->sessions, value.value);
4007   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4008   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4009   return 0;
4010 }
4011
4012 void
4013 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4014 {
4015   snat_main_t *sm = &snat_main;
4016
4017   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4018   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4019   sm->psid = psid;
4020   sm->psid_offset = psid_offset;
4021   sm->psid_length = psid_length;
4022 }
4023
4024 void
4025 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4026 {
4027   snat_main_t *sm = &snat_main;
4028
4029   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4030   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4031   sm->start_port = start_port;
4032   sm->end_port = end_port;
4033 }
4034
4035 void
4036 nat_set_alloc_addr_and_port_default (void)
4037 {
4038   snat_main_t *sm = &snat_main;
4039
4040   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4041   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4042 }
4043
4044 /*
4045  * fd.io coding-style-patch-verification: ON
4046  *
4047  * Local Variables:
4048  * eval: (c-set-style "gnu")
4049  * End:
4050  */