src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <vnet/fib/fib_table.h>
  35 #include <vnet/fib/ip4_fib.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 /* *INDENT-OFF* */
  42
  43 /* Hook up input features */
  44 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  45   .arc_name = "ip4-unicast",
  46   .node_name = "nat44-in2out",
  47   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  48 };
  49 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  50   .arc_name = "ip4-unicast",
  51   .node_name = "nat44-out2in",
  52   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  53                                "ip4-dhcp-client-detect"),
  54 };
  55 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  56   .arc_name = "ip4-unicast",
  57   .node_name = "nat44-classify",
  58   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  59 };
  60 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  61   .arc_name = "ip4-unicast",
  62   .node_name = "nat44-det-in2out",
  63   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  64 };
  65 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  66   .arc_name = "ip4-unicast",
  67   .node_name = "nat44-det-out2in",
  68   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  69                                "ip4-dhcp-client-detect"),
  70 };
  71 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  72   .arc_name = "ip4-unicast",
  73   .node_name = "nat44-det-classify",
  74   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  75 };
  76 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  77   .arc_name = "ip4-unicast",
  78   .node_name = "nat44-ed-in2out",
  79   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-ed-out2in",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  85                                "ip4-dhcp-client-detect"),
  86 };
  87 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  88   .arc_name = "ip4-unicast",
  89   .node_name = "nat44-ed-classify",
  90   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  91 };
  92 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-in2out-worker-handoff",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  96 };
  97 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-out2in-worker-handoff",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 101                                "ip4-dhcp-client-detect"),
 102 };
 103 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-handoff-classify",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 107 };
 108 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-in2out-fast",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 112 };
 113 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-out2in-fast",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 117                                "ip4-dhcp-client-detect"),
 118 };
 119 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-hairpin-dst",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 123 };
 124 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-ed-hairpin-dst",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 128 };
 129
 130 /* Hook up output features */
 131 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 132   .arc_name = "ip4-output",
 133   .node_name = "nat44-in2out-output",
 134   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 137   .arc_name = "ip4-output",
 138   .node_name = "nat44-in2out-output-worker-handoff",
 139   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 140 };
 141 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 142   .arc_name = "ip4-output",
 143   .node_name = "nat44-hairpin-src",
 144   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 145 };
 146 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-ed-in2out-output",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 150 };
 151 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-ed-hairpin-src",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 155 };
 156
 157 /* Hook up ip4-local features */
 158 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 159 {
 160   .arc_name = "ip4-local",
 161   .node_name = "nat44-hairpinning",
 162   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 163 };
 164 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 165 {
 166   .arc_name = "ip4-local",
 167   .node_name = "nat44-ed-hairpinning",
 168   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 169 };
 170
 171
 172 VLIB_PLUGIN_REGISTER () = {
 173     .version = VPP_BUILD_VER,
 174     .description = "Network Address Translation",
 175 };
 176 /* *INDENT-ON* */
 177
 178 void
 179 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 180                        u8 is_ha)
 181 {
 182   snat_session_key_t key;
 183   clib_bihash_kv_8_8_t kv;
 184   nat_ed_ses_key_t ed_key;
 185   clib_bihash_kv_16_8_t ed_kv;
 186   snat_main_per_thread_data_t *tsm =
 187     vec_elt_at_index (sm->per_thread_data, thread_index);
 188
 189   if (is_fwd_bypass_session (s))
 190     {
 191       ed_key.l_addr = s->in2out.addr;
 192       ed_key.r_addr = s->ext_host_addr;
 193       ed_key.l_port = s->in2out.port;
 194       ed_key.r_port = s->ext_host_port;
 195       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 196       ed_key.fib_index = 0;
 197       ed_kv.key[0] = ed_key.as_u64[0];
 198       ed_kv.key[1] = ed_key.as_u64[1];
 199       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 200         nat_log_warn ("in2out_ed key del failed");
 201       return;
 202     }
 203
 204   /* session lookup tables */
 205   if (is_ed_session (s))
 206     {
 207       if (is_affinity_sessions (s))
 208         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 209                              s->in2out.protocol, s->out2in.port);
 210       ed_key.l_addr = s->out2in.addr;
 211       ed_key.r_addr = s->ext_host_addr;
 212       ed_key.fib_index = s->out2in.fib_index;
 213       if (snat_is_unk_proto_session (s))
 214         {
 215           ed_key.proto = s->in2out.port;
 216           ed_key.r_port = 0;
 217           ed_key.l_port = 0;
 218         }
 219       else
 220         {
 221           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 222           ed_key.l_port = s->out2in.port;
 223           ed_key.r_port = s->ext_host_port;
 224         }
 225       ed_kv.key[0] = ed_key.as_u64[0];
 226       ed_kv.key[1] = ed_key.as_u64[1];
 227       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 228         nat_log_warn ("out2in_ed key del failed");
 229       ed_key.l_addr = s->in2out.addr;
 230       ed_key.fib_index = s->in2out.fib_index;
 231       if (!snat_is_unk_proto_session (s))
 232         ed_key.l_port = s->in2out.port;
 233       if (is_twice_nat_session (s))
 234         {
 235           ed_key.r_addr = s->ext_host_nat_addr;
 236           ed_key.r_port = s->ext_host_nat_port;
 237         }
 238       ed_kv.key[0] = ed_key.as_u64[0];
 239       ed_kv.key[1] = ed_key.as_u64[1];
 240       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 241         nat_log_warn ("in2out_ed key del failed");
 242
 243       if (!is_ha)
 244         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 245                                &s->in2out.addr, s->in2out.port,
 246                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 247                                &s->out2in.addr, s->out2in.port,
 248                                &s->ext_host_addr, s->ext_host_port,
 249                                s->in2out.protocol, is_twice_nat_session (s));
 250     }
 251   else
 252     {
 253       kv.key = s->in2out.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 255         nat_log_warn ("in2out key del failed");
 256       kv.key = s->out2in.as_u64;
 257       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 258         nat_log_warn ("out2in key del failed");
 259
 260       if (!is_ha)
 261         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 262                                  &s->in2out.addr, s->in2out.port,
 263                                  &s->out2in.addr, s->out2in.port,
 264                                  s->in2out.protocol);
 265     }
 266
 267   if (snat_is_unk_proto_session (s))
 268     return;
 269
 270   if (!is_ha)
 271     {
 272       /* log NAT event */
 273       snat_ipfix_logging_nat44_ses_delete (thread_index,
 274                                            s->in2out.addr.as_u32,
 275                                            s->out2in.addr.as_u32,
 276                                            s->in2out.protocol,
 277                                            s->in2out.port,
 278                                            s->out2in.port,
 279                                            s->in2out.fib_index);
 280
 281       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 282                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 283                    thread_index);
 284     }
 285
 286   /* Twice NAT address and port for external host */
 287   if (is_twice_nat_session (s))
 288     {
 289       key.protocol = s->in2out.protocol;
 290       key.port = s->ext_host_nat_port;
 291       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 292       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 293                                           thread_index, &key);
 294     }
 295
 296   if (snat_is_session_static (s))
 297     return;
 298
 299   snat_free_outside_address_and_port (sm->addresses, thread_index,
 300                                       &s->out2in);
 301 }
 302
 303 snat_user_t *
 304 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 305                         u32 thread_index)
 306 {
 307   snat_user_t *u = 0;
 308   snat_user_key_t user_key;
 309   clib_bihash_kv_8_8_t kv, value;
 310   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 311   dlist_elt_t *per_user_list_head_elt;
 312
 313   user_key.addr.as_u32 = addr->as_u32;
 314   user_key.fib_index = fib_index;
 315   kv.key = user_key.as_u64;
 316
 317   /* Ever heard of the "user" = src ip4 address before? */
 318   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 319     {
 320       /* no, make a new one */
 321       pool_get (tsm->users, u);
 322       clib_memset (u, 0, sizeof (*u));
 323       u->addr.as_u32 = addr->as_u32;
 324       u->fib_index = fib_index;
 325
 326       pool_get (tsm->list_pool, per_user_list_head_elt);
 327
 328       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 329         tsm->list_pool;
 330
 331       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 332
 333       kv.value = u - tsm->users;
 334
 335       /* add user */
 336       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 337         nat_log_warn ("user_hash keay add failed");
 338
 339       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 340                                pool_elts (tsm->users));
 341     }
 342   else
 343     {
 344       u = pool_elt_at_index (tsm->users, value.value);
 345     }
 346
 347   return u;
 348 }
 349
 350 snat_session_t *
 351 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 352                               u32 thread_index, f64 now)
 353 {
 354   snat_session_t *s;
 355   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 356   u32 oldest_per_user_translation_list_index, session_index;
 357   dlist_elt_t *oldest_per_user_translation_list_elt;
 358   dlist_elt_t *per_user_translation_list_elt;
 359
 360   /* Over quota? Recycle the least recently used translation */
 361   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 362     {
 363       oldest_per_user_translation_list_index =
 364         clib_dlist_remove_head (tsm->list_pool,
 365                                 u->sessions_per_user_list_head_index);
 366
 367       ASSERT (oldest_per_user_translation_list_index != ~0);
 368
 369       /* Add it back to the end of the LRU list */
 370       clib_dlist_addtail (tsm->list_pool,
 371                           u->sessions_per_user_list_head_index,
 372                           oldest_per_user_translation_list_index);
 373       /* Get the list element */
 374       oldest_per_user_translation_list_elt =
 375         pool_elt_at_index (tsm->list_pool,
 376                            oldest_per_user_translation_list_index);
 377
 378       /* Get the session index from the list element */
 379       session_index = oldest_per_user_translation_list_elt->value;
 380
 381       /* Get the session */
 382       s = pool_elt_at_index (tsm->sessions, session_index);
 383       nat_free_session_data (sm, s, thread_index, 0);
 384       if (snat_is_session_static (s))
 385         u->nstaticsessions--;
 386       else
 387         u->nsessions--;
 388       s->flags = 0;
 389       s->total_bytes = 0;
 390       s->total_pkts = 0;
 391       s->state = 0;
 392       s->ext_host_addr.as_u32 = 0;
 393       s->ext_host_port = 0;
 394       s->ext_host_nat_addr.as_u32 = 0;
 395       s->ext_host_nat_port = 0;
 396     }
 397   else
 398     {
 399       pool_get (tsm->sessions, s);
 400       clib_memset (s, 0, sizeof (*s));
 401
 402       /* Create list elts */
 403       pool_get (tsm->list_pool, per_user_translation_list_elt);
 404       clib_dlist_init (tsm->list_pool,
 405                        per_user_translation_list_elt - tsm->list_pool);
 406
 407       per_user_translation_list_elt->value = s - tsm->sessions;
 408       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 409       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 410
 411       clib_dlist_addtail (tsm->list_pool,
 412                           s->per_user_list_head_index,
 413                           per_user_translation_list_elt - tsm->list_pool);
 414
 415       s->user_index = u - tsm->users;
 416       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 417                                pool_elts (tsm->sessions));
 418     }
 419
 420   s->ha_last_refreshed = now;
 421
 422   return s;
 423 }
 424
 425 snat_session_t *
 426 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 427                       f64 now)
 428 {
 429   snat_session_t *s;
 430   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 431   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 432   u32 oldest_index;
 433   u64 sess_timeout_time;
 434
 435   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 436     goto alloc_new;
 437
 438   oldest_index =
 439     clib_dlist_remove_head (tsm->list_pool,
 440                             u->sessions_per_user_list_head_index);
 441   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 442   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 443   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 444   if (now >= sess_timeout_time)
 445     {
 446       clib_dlist_addtail (tsm->list_pool,
 447                           u->sessions_per_user_list_head_index, oldest_index);
 448       nat_free_session_data (sm, s, thread_index, 0);
 449       if (snat_is_session_static (s))
 450         u->nstaticsessions--;
 451       else
 452         u->nsessions--;
 453       s->flags = 0;
 454       s->total_bytes = 0;
 455       s->total_pkts = 0;
 456       s->state = 0;
 457       s->ext_host_addr.as_u32 = 0;
 458       s->ext_host_port = 0;
 459       s->ext_host_nat_addr.as_u32 = 0;
 460       s->ext_host_nat_port = 0;
 461     }
 462   else
 463     {
 464       clib_dlist_addhead (tsm->list_pool,
 465                           u->sessions_per_user_list_head_index, oldest_index);
 466       if ((u->nsessions + u->nstaticsessions) >=
 467           sm->max_translations_per_user)
 468         {
 469           nat_log_warn ("max translations per user %U", format_ip4_address,
 470                         &u->addr);
 471           snat_ipfix_logging_max_entries_per_user
 472             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 473           return 0;
 474         }
 475       else
 476         {
 477         alloc_new:
 478           pool_get (tsm->sessions, s);
 479           clib_memset (s, 0, sizeof (*s));
 480
 481           /* Create list elts */
 482           pool_get (tsm->list_pool, per_user_translation_list_elt);
 483           clib_dlist_init (tsm->list_pool,
 484                            per_user_translation_list_elt - tsm->list_pool);
 485
 486           per_user_translation_list_elt->value = s - tsm->sessions;
 487           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 488           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 489
 490           clib_dlist_addtail (tsm->list_pool,
 491                               s->per_user_list_head_index,
 492                               per_user_translation_list_elt - tsm->list_pool);
 493         }
 494
 495       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 496                                pool_elts (tsm->sessions));
 497     }
 498
 499   s->ha_last_refreshed = now;
 500
 501   return s;
 502 }
 503
 504 void
 505 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 506                           int is_add)
 507 {
 508   fib_prefix_t prefix = {
 509     .fp_len = p_len,
 510     .fp_proto = FIB_PROTOCOL_IP4,
 511     .fp_addr = {
 512                 .ip4.as_u32 = addr->as_u32,
 513                 },
 514   };
 515   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 516
 517   if (is_add)
 518     fib_table_entry_update_one_path (fib_index,
 519                                      &prefix,
 520                                      FIB_SOURCE_PLUGIN_LOW,
 521                                      (FIB_ENTRY_FLAG_CONNECTED |
 522                                       FIB_ENTRY_FLAG_LOCAL |
 523                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 524                                      DPO_PROTO_IP4,
 525                                      NULL,
 526                                      sw_if_index,
 527                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 528   else
 529     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 530 }
 531
 532 int
 533 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 534                   u8 twice_nat)
 535 {
 536   snat_address_t *ap;
 537   snat_interface_t *i;
 538   vlib_thread_main_t *tm = vlib_get_thread_main ();
 539
 540   if (twice_nat && !sm->endpoint_dependent)
 541     return VNET_API_ERROR_FEATURE_DISABLED;
 542
 543   /* Check if address already exists */
 544   /* *INDENT-OFF* */
 545   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 546     {
 547       if (ap->addr.as_u32 == addr->as_u32)
 548         return VNET_API_ERROR_VALUE_EXIST;
 549     }
 550   /* *INDENT-ON* */
 551
 552   if (twice_nat)
 553     vec_add2 (sm->twice_nat_addresses, ap, 1);
 554   else
 555     vec_add2 (sm->addresses, ap, 1);
 556
 557   ap->addr = *addr;
 558   if (vrf_id != ~0)
 559     ap->fib_index =
 560       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 561                                          FIB_SOURCE_PLUGIN_LOW);
 562   else
 563     ap->fib_index = ~0;
 564 #define _(N, i, n, s) \
 565   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 566   ap->busy_##n##_ports = 0; \
 567   ap->busy_##n##_ports_per_thread = 0;\
 568   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 569   foreach_snat_protocol
 570 #undef _
 571     if (twice_nat)
 572     return 0;
 573
 574   /* Add external address to FIB */
 575   /* *INDENT-OFF* */
 576   pool_foreach (i, sm->interfaces,
 577   ({
 578     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 579       continue;
 580
 581     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 582     break;
 583   }));
 584   pool_foreach (i, sm->output_feature_interfaces,
 585   ({
 586     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 587       continue;
 588
 589     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 590     break;
 591   }));
 592   /* *INDENT-ON* */
 593
 594   return 0;
 595 }
 596
 597 static int
 598 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 599 {
 600   snat_static_mapping_t *m;
 601   /* *INDENT-OFF* */
 602   pool_foreach (m, sm->static_mappings,
 603   ({
 604       if (is_addr_only_static_mapping (m) ||
 605           is_out2in_only_static_mapping (m) ||
 606           is_identity_static_mapping (m))
 607         continue;
 608       if (m->external_addr.as_u32 == addr.as_u32)
 609         return 1;
 610   }));
 611   /* *INDENT-ON* */
 612
 613   return 0;
 614 }
 615
 616 void
 617 increment_v4_address (ip4_address_t * a)
 618 {
 619   u32 v;
 620
 621   v = clib_net_to_host_u32 (a->as_u32) + 1;
 622   a->as_u32 = clib_host_to_net_u32 (v);
 623 }
 624
 625 static void
 626 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 627                                        ip4_address_t l_addr,
 628                                        u16 l_port,
 629                                        u32 sw_if_index,
 630                                        u16 e_port,
 631                                        u32 vrf_id,
 632                                        snat_protocol_t proto,
 633                                        int addr_only, int is_add, u8 * tag,
 634                                        int twice_nat, int out2in_only,
 635                                        int identity_nat)
 636 {
 637   snat_static_map_resolve_t *rp;
 638
 639   vec_add2 (sm->to_resolve, rp, 1);
 640   rp->l_addr.as_u32 = l_addr.as_u32;
 641   rp->l_port = l_port;
 642   rp->sw_if_index = sw_if_index;
 643   rp->e_port = e_port;
 644   rp->vrf_id = vrf_id;
 645   rp->proto = proto;
 646   rp->addr_only = addr_only;
 647   rp->is_add = is_add;
 648   rp->twice_nat = twice_nat;
 649   rp->out2in_only = out2in_only;
 650   rp->identity_nat = identity_nat;
 651   rp->tag = vec_dup (tag);
 652 }
 653
 654 static u32
 655 get_thread_idx_by_port (u16 e_port)
 656 {
 657   snat_main_t *sm = &snat_main;
 658   u32 thread_idx = sm->num_workers;
 659   if (sm->num_workers > 1)
 660     {
 661       thread_idx =
 662         sm->first_worker_index +
 663         sm->workers[(e_port - 1024) / sm->port_per_thread];
 664     }
 665   return thread_idx;
 666 }
 667
 668 int
 669 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 670                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 671                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 672                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 673                          u8 identity_nat)
 674 {
 675   snat_main_t *sm = &snat_main;
 676   snat_static_mapping_t *m;
 677   snat_session_key_t m_key;
 678   clib_bihash_kv_8_8_t kv, value;
 679   snat_address_t *a = 0;
 680   u32 fib_index = ~0;
 681   snat_interface_t *interface;
 682   int i;
 683   snat_main_per_thread_data_t *tsm;
 684   snat_user_key_t u_key;
 685   snat_user_t *u;
 686   dlist_elt_t *head, *elt;
 687   u32 elt_index, head_index;
 688   u32 ses_index;
 689   u64 user_index;
 690   snat_session_t *s;
 691   snat_static_map_resolve_t *rp, *rp_match = 0;
 692   nat44_lb_addr_port_t *local;
 693   u32 find = ~0;
 694
 695   if (!sm->endpoint_dependent)
 696     {
 697       if (twice_nat || out2in_only)
 698         return VNET_API_ERROR_FEATURE_DISABLED;
 699     }
 700
 701   /* If the external address is a specific interface address */
 702   if (sw_if_index != ~0)
 703     {
 704       ip4_address_t *first_int_addr;
 705
 706       for (i = 0; i < vec_len (sm->to_resolve); i++)
 707         {
 708           rp = sm->to_resolve + i;
 709           if (rp->sw_if_index != sw_if_index ||
 710               rp->l_addr.as_u32 != l_addr.as_u32 ||
 711               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 712             continue;
 713
 714           if (!addr_only)
 715             {
 716               if (rp->l_port != l_port || rp->e_port != e_port
 717                   || rp->proto != proto)
 718                 continue;
 719             }
 720
 721           rp_match = rp;
 722           break;
 723         }
 724
 725       /* Might be already set... */
 726       first_int_addr = ip4_interface_first_address
 727         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 728
 729       if (is_add)
 730         {
 731           if (rp_match)
 732             return VNET_API_ERROR_VALUE_EXIST;
 733
 734           snat_add_static_mapping_when_resolved
 735             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 736              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 737
 738           /* DHCP resolution required? */
 739           if (first_int_addr == 0)
 740             {
 741               return 0;
 742             }
 743           else
 744             {
 745               e_addr.as_u32 = first_int_addr->as_u32;
 746               /* Identity mapping? */
 747               if (l_addr.as_u32 == 0)
 748                 l_addr.as_u32 = e_addr.as_u32;
 749             }
 750         }
 751       else
 752         {
 753           if (!rp_match)
 754             return VNET_API_ERROR_NO_SUCH_ENTRY;
 755
 756           vec_del1 (sm->to_resolve, i);
 757
 758           if (first_int_addr)
 759             {
 760               e_addr.as_u32 = first_int_addr->as_u32;
 761               /* Identity mapping? */
 762               if (l_addr.as_u32 == 0)
 763                 l_addr.as_u32 = e_addr.as_u32;
 764             }
 765           else
 766             return 0;
 767         }
 768     }
 769
 770   m_key.addr = e_addr;
 771   m_key.port = addr_only ? 0 : e_port;
 772   m_key.protocol = addr_only ? 0 : proto;
 773   m_key.fib_index = 0;
 774   kv.key = m_key.as_u64;
 775   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 776     m = 0;
 777   else
 778     m = pool_elt_at_index (sm->static_mappings, value.value);
 779
 780   if (is_add)
 781     {
 782       if (m)
 783         {
 784           if (is_identity_static_mapping (m))
 785             {
 786               /* *INDENT-OFF* */
 787               pool_foreach (local, m->locals,
 788               ({
 789                 if (local->vrf_id == vrf_id)
 790                   return VNET_API_ERROR_VALUE_EXIST;
 791               }));
 792               /* *INDENT-ON* */
 793               pool_get (m->locals, local);
 794               local->vrf_id = vrf_id;
 795               local->fib_index =
 796                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 797                                                    FIB_SOURCE_PLUGIN_LOW);
 798               m_key.addr = m->local_addr;
 799               m_key.port = m->local_port;
 800               m_key.protocol = m->proto;
 801               m_key.fib_index = local->fib_index;
 802               kv.key = m_key.as_u64;
 803               kv.value = m - sm->static_mappings;
 804               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 805               return 0;
 806             }
 807           else
 808             return VNET_API_ERROR_VALUE_EXIST;
 809         }
 810
 811       if (twice_nat && addr_only)
 812         return VNET_API_ERROR_UNSUPPORTED;
 813
 814       /* Convert VRF id to FIB index */
 815       if (vrf_id != ~0)
 816         fib_index =
 817           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 818                                              FIB_SOURCE_PLUGIN_LOW);
 819       /* If not specified use inside VRF id from SNAT plugin startup config */
 820       else
 821         {
 822           fib_index = sm->inside_fib_index;
 823           vrf_id = sm->inside_vrf_id;
 824         }
 825
 826       if (!(out2in_only || identity_nat))
 827         {
 828           m_key.addr = l_addr;
 829           m_key.port = addr_only ? 0 : l_port;
 830           m_key.protocol = addr_only ? 0 : proto;
 831           m_key.fib_index = fib_index;
 832           kv.key = m_key.as_u64;
 833           if (!clib_bihash_search_8_8
 834               (&sm->static_mapping_by_local, &kv, &value))
 835             return VNET_API_ERROR_VALUE_EXIST;
 836         }
 837
 838       /* Find external address in allocated addresses and reserve port for
 839          address and port pair mapping when dynamic translations enabled */
 840       if (!(addr_only || sm->static_mapping_only || out2in_only))
 841         {
 842           for (i = 0; i < vec_len (sm->addresses); i++)
 843             {
 844               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 845                 {
 846                   a = sm->addresses + i;
 847                   /* External port must be unused */
 848                   switch (proto)
 849                     {
 850 #define _(N, j, n, s) \
 851                     case SNAT_PROTOCOL_##N: \
 852                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 853                         return VNET_API_ERROR_INVALID_VALUE; \
 854                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 855                       if (e_port > 1024) \
 856                         { \
 857                           a->busy_##n##_ports++; \
 858                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 859                         } \
 860                       break;
 861                       foreach_snat_protocol
 862 #undef _
 863                     default:
 864                       nat_log_info ("unknown protocol");
 865                       return VNET_API_ERROR_INVALID_VALUE_2;
 866                     }
 867                   break;
 868                 }
 869             }
 870           /* External address must be allocated */
 871           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 872             {
 873               if (sw_if_index != ~0)
 874                 {
 875                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 876                     {
 877                       rp = sm->to_resolve + i;
 878                       if (rp->addr_only)
 879                         continue;
 880                       if (rp->sw_if_index != sw_if_index &&
 881                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 882                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 883                           rp->e_port != e_port && rp->proto != proto)
 884                         continue;
 885
 886                       vec_del1 (sm->to_resolve, i);
 887                       break;
 888                     }
 889                 }
 890               return VNET_API_ERROR_NO_SUCH_ENTRY;
 891             }
 892         }
 893
 894       pool_get (sm->static_mappings, m);
 895       clib_memset (m, 0, sizeof (*m));
 896       m->tag = vec_dup (tag);
 897       m->local_addr = l_addr;
 898       m->external_addr = e_addr;
 899       m->twice_nat = twice_nat;
 900       if (out2in_only)
 901         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 902       if (addr_only)
 903         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 904       if (identity_nat)
 905         {
 906           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 907           pool_get (m->locals, local);
 908           local->vrf_id = vrf_id;
 909           local->fib_index = fib_index;
 910         }
 911       else
 912         {
 913           m->vrf_id = vrf_id;
 914           m->fib_index = fib_index;
 915         }
 916       if (!addr_only)
 917         {
 918           m->local_port = l_port;
 919           m->external_port = e_port;
 920           m->proto = proto;
 921         }
 922
 923       if (sm->num_workers > 1)
 924         {
 925           ip4_header_t ip = {
 926             .src_address = m->local_addr,
 927           };
 928           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 929           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 930         }
 931       else
 932         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 933
 934       m_key.addr = m->local_addr;
 935       m_key.port = m->local_port;
 936       m_key.protocol = m->proto;
 937       m_key.fib_index = fib_index;
 938       kv.key = m_key.as_u64;
 939       kv.value = m - sm->static_mappings;
 940       if (!out2in_only)
 941         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 942
 943       m_key.addr = m->external_addr;
 944       m_key.port = m->external_port;
 945       m_key.fib_index = 0;
 946       kv.key = m_key.as_u64;
 947       kv.value = m - sm->static_mappings;
 948       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 949
 950       /* Delete dynamic sessions matching local address (+ local port) */
 951       if (!(sm->static_mapping_only))
 952         {
 953           u_key.addr = m->local_addr;
 954           u_key.fib_index = m->fib_index;
 955           kv.key = u_key.as_u64;
 956           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 957             {
 958               user_index = value.value;
 959               u = pool_elt_at_index (tsm->users, user_index);
 960               if (u->nsessions)
 961                 {
 962                   head_index = u->sessions_per_user_list_head_index;
 963                   head = pool_elt_at_index (tsm->list_pool, head_index);
 964                   elt_index = head->next;
 965                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 966                   ses_index = elt->value;
 967                   while (ses_index != ~0)
 968                     {
 969                       s = pool_elt_at_index (tsm->sessions, ses_index);
 970                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 971                       ses_index = elt->value;
 972
 973                       if (snat_is_session_static (s))
 974                         continue;
 975
 976                       if (!addr_only
 977                           && (clib_net_to_host_u16 (s->in2out.port) !=
 978                               m->local_port))
 979                         continue;
 980
 981                       nat_free_session_data (sm, s,
 982                                              tsm - sm->per_thread_data, 0);
 983                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 984
 985                       if (!addr_only && !sm->endpoint_dependent)
 986                         break;
 987                     }
 988                 }
 989             }
 990         }
 991     }
 992   else
 993     {
 994       if (!m)
 995         {
 996           if (sw_if_index != ~0)
 997             return 0;
 998           else
 999             return VNET_API_ERROR_NO_SUCH_ENTRY;
1000         }
1001
1002       if (identity_nat)
1003         {
1004           if (vrf_id == ~0)
1005             vrf_id = sm->inside_vrf_id;
1006
1007           /* *INDENT-OFF* */
1008           pool_foreach (local, m->locals,
1009           ({
1010             if (local->vrf_id == vrf_id)
1011               find = local - m->locals;
1012           }));
1013           /* *INDENT-ON* */
1014           if (find == ~0)
1015             return VNET_API_ERROR_NO_SUCH_ENTRY;
1016
1017           local = pool_elt_at_index (m->locals, find);
1018           fib_index = local->fib_index;
1019           pool_put (m->locals, local);
1020         }
1021       else
1022         fib_index = m->fib_index;
1023
1024       /* Free external address port */
1025       if (!(addr_only || sm->static_mapping_only || out2in_only))
1026         {
1027           for (i = 0; i < vec_len (sm->addresses); i++)
1028             {
1029               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1030                 {
1031                   a = sm->addresses + i;
1032                   switch (proto)
1033                     {
1034 #define _(N, j, n, s) \
1035                     case SNAT_PROTOCOL_##N: \
1036                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1037                       if (e_port > 1024) \
1038                         { \
1039                           a->busy_##n##_ports--; \
1040                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1041                         } \
1042                       break;
1043                       foreach_snat_protocol
1044 #undef _
1045                     default:
1046                       nat_log_info ("unknown protocol");
1047                       return VNET_API_ERROR_INVALID_VALUE_2;
1048                     }
1049                   break;
1050                 }
1051             }
1052         }
1053
1054       if (sm->num_workers > 1)
1055         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1056       else
1057         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1058
1059       m_key.addr = m->local_addr;
1060       m_key.port = m->local_port;
1061       m_key.protocol = m->proto;
1062       m_key.fib_index = fib_index;
1063       kv.key = m_key.as_u64;
1064       if (!out2in_only)
1065         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1066
1067       /* Delete session(s) for static mapping if exist */
1068       if (!(sm->static_mapping_only) ||
1069           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1070         {
1071           u_key.addr = m->local_addr;
1072           u_key.fib_index = fib_index;
1073           kv.key = u_key.as_u64;
1074           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1075             {
1076               user_index = value.value;
1077               u = pool_elt_at_index (tsm->users, user_index);
1078               if (u->nstaticsessions)
1079                 {
1080                   head_index = u->sessions_per_user_list_head_index;
1081                   head = pool_elt_at_index (tsm->list_pool, head_index);
1082                   elt_index = head->next;
1083                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1084                   ses_index = elt->value;
1085                   while (ses_index != ~0)
1086                     {
1087                       s = pool_elt_at_index (tsm->sessions, ses_index);
1088                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1089                       ses_index = elt->value;
1090
1091                       if (!addr_only)
1092                         {
1093                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1094                               (clib_net_to_host_u16 (s->out2in.port) !=
1095                                e_port))
1096                             continue;
1097                         }
1098
1099                       if (is_lb_session (s))
1100                         continue;
1101
1102                       if (!snat_is_session_static (s))
1103                         continue;
1104
1105                       nat_free_session_data (sm, s,
1106                                              tsm - sm->per_thread_data, 0);
1107                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1108
1109                       if (!addr_only && !sm->endpoint_dependent)
1110                         break;
1111                     }
1112                 }
1113             }
1114         }
1115
1116       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1117       if (pool_elts (m->locals))
1118         return 0;
1119
1120       m_key.addr = m->external_addr;
1121       m_key.port = m->external_port;
1122       m_key.fib_index = 0;
1123       kv.key = m_key.as_u64;
1124       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1125
1126       vec_free (m->tag);
1127       vec_free (m->workers);
1128       /* Delete static mapping from pool */
1129       pool_put (sm->static_mappings, m);
1130     }
1131
1132   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1133     return 0;
1134
1135   /* Add/delete external address to FIB */
1136   /* *INDENT-OFF* */
1137   pool_foreach (interface, sm->interfaces,
1138   ({
1139     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1140       continue;
1141
1142     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1143     break;
1144   }));
1145   pool_foreach (interface, sm->output_feature_interfaces,
1146   ({
1147     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1148       continue;
1149
1150     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1151     break;
1152   }));
1153   /* *INDENT-ON* */
1154
1155   return 0;
1156 }
1157
1158 int
1159 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1160                                  snat_protocol_t proto,
1161                                  nat44_lb_addr_port_t * locals, u8 is_add,
1162                                  twice_nat_type_t twice_nat, u8 out2in_only,
1163                                  u8 * tag, u32 affinity)
1164 {
1165   snat_main_t *sm = &snat_main;
1166   snat_static_mapping_t *m;
1167   snat_session_key_t m_key;
1168   clib_bihash_kv_8_8_t kv, value;
1169   snat_address_t *a = 0;
1170   int i;
1171   nat44_lb_addr_port_t *local;
1172   u32 elt_index, head_index, ses_index;
1173   snat_main_per_thread_data_t *tsm;
1174   snat_user_key_t u_key;
1175   snat_user_t *u;
1176   snat_session_t *s;
1177   dlist_elt_t *head, *elt;
1178   uword *bitmap = 0;
1179
1180   if (!sm->endpoint_dependent)
1181     return VNET_API_ERROR_FEATURE_DISABLED;
1182
1183   m_key.addr = e_addr;
1184   m_key.port = e_port;
1185   m_key.protocol = proto;
1186   m_key.fib_index = 0;
1187   kv.key = m_key.as_u64;
1188   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1189     m = 0;
1190   else
1191     m = pool_elt_at_index (sm->static_mappings, value.value);
1192
1193   if (is_add)
1194     {
1195       if (m)
1196         return VNET_API_ERROR_VALUE_EXIST;
1197
1198       if (vec_len (locals) < 2)
1199         return VNET_API_ERROR_INVALID_VALUE;
1200
1201       /* Find external address in allocated addresses and reserve port for
1202          address and port pair mapping when dynamic translations enabled */
1203       if (!(sm->static_mapping_only || out2in_only))
1204         {
1205           for (i = 0; i < vec_len (sm->addresses); i++)
1206             {
1207               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1208                 {
1209                   a = sm->addresses + i;
1210                   /* External port must be unused */
1211                   switch (proto)
1212                     {
1213 #define _(N, j, n, s) \
1214                     case SNAT_PROTOCOL_##N: \
1215                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1216                         return VNET_API_ERROR_INVALID_VALUE; \
1217                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1218                       if (e_port > 1024) \
1219                         { \
1220                           a->busy_##n##_ports++; \
1221                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1222                         } \
1223                       break;
1224                       foreach_snat_protocol
1225 #undef _
1226                     default:
1227                       nat_log_info ("unknown protocol");
1228                       return VNET_API_ERROR_INVALID_VALUE_2;
1229                     }
1230                   break;
1231                 }
1232             }
1233           /* External address must be allocated */
1234           if (!a)
1235             return VNET_API_ERROR_NO_SUCH_ENTRY;
1236         }
1237
1238       pool_get (sm->static_mappings, m);
1239       clib_memset (m, 0, sizeof (*m));
1240       m->tag = vec_dup (tag);
1241       m->external_addr = e_addr;
1242       m->external_port = e_port;
1243       m->proto = proto;
1244       m->twice_nat = twice_nat;
1245       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1246       if (out2in_only)
1247         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1248       m->affinity = affinity;
1249
1250       if (affinity)
1251         m->affinity_per_service_list_head_index =
1252           nat_affinity_get_per_service_list_head_index ();
1253       else
1254         m->affinity_per_service_list_head_index = ~0;
1255
1256       m_key.addr = m->external_addr;
1257       m_key.port = m->external_port;
1258       m_key.protocol = m->proto;
1259       m_key.fib_index = 0;
1260       kv.key = m_key.as_u64;
1261       kv.value = m - sm->static_mappings;
1262       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1263         {
1264           nat_log_err ("static_mapping_by_external key add failed");
1265           return VNET_API_ERROR_UNSPECIFIED;
1266         }
1267
1268       m_key.fib_index = m->fib_index;
1269       for (i = 0; i < vec_len (locals); i++)
1270         {
1271           locals[i].fib_index =
1272             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1273                                                locals[i].vrf_id,
1274                                                FIB_SOURCE_PLUGIN_LOW);
1275           m_key.addr = locals[i].addr;
1276           m_key.fib_index = locals[i].fib_index;
1277           if (!out2in_only)
1278             {
1279               m_key.port = locals[i].port;
1280               kv.key = m_key.as_u64;
1281               kv.value = m - sm->static_mappings;
1282               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1283             }
1284           locals[i].prefix = (i == 0) ? locals[i].probability :
1285             (locals[i - 1].prefix + locals[i].probability);
1286           pool_get (m->locals, local);
1287           *local = locals[i];
1288           if (sm->num_workers > 1)
1289             {
1290               ip4_header_t ip = {
1291                 .src_address = locals[i].addr,
1292               };
1293               bitmap =
1294                 clib_bitmap_set (bitmap,
1295                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1296             }
1297         }
1298
1299       /* Assign workers */
1300       if (sm->num_workers > 1)
1301         {
1302           /* *INDENT-OFF* */
1303           clib_bitmap_foreach (i, bitmap,
1304             ({
1305                vec_add1(m->workers, i);
1306             }));
1307           /* *INDENT-ON* */
1308         }
1309     }
1310   else
1311     {
1312       if (!m)
1313         return VNET_API_ERROR_NO_SUCH_ENTRY;
1314
1315       if (!is_lb_static_mapping (m))
1316         return VNET_API_ERROR_INVALID_VALUE;
1317
1318       /* Free external address port */
1319       if (!(sm->static_mapping_only || out2in_only))
1320         {
1321           for (i = 0; i < vec_len (sm->addresses); i++)
1322             {
1323               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1324                 {
1325                   a = sm->addresses + i;
1326                   switch (proto)
1327                     {
1328 #define _(N, j, n, s) \
1329                     case SNAT_PROTOCOL_##N: \
1330                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1331                       if (e_port > 1024) \
1332                         { \
1333                           a->busy_##n##_ports--; \
1334                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1335                         } \
1336                       break;
1337                       foreach_snat_protocol
1338 #undef _
1339                     default:
1340                       nat_log_info ("unknown protocol");
1341                       return VNET_API_ERROR_INVALID_VALUE_2;
1342                     }
1343                   break;
1344                 }
1345             }
1346         }
1347
1348       m_key.addr = m->external_addr;
1349       m_key.port = m->external_port;
1350       m_key.protocol = m->proto;
1351       m_key.fib_index = 0;
1352       kv.key = m_key.as_u64;
1353       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1354         {
1355           nat_log_err ("static_mapping_by_external key del failed");
1356           return VNET_API_ERROR_UNSPECIFIED;
1357         }
1358
1359       /* *INDENT-OFF* */
1360       pool_foreach (local, m->locals,
1361       ({
1362           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1363                             FIB_SOURCE_PLUGIN_LOW);
1364           m_key.addr = local->addr;
1365           if (!out2in_only)
1366             {
1367               m_key.port = local->port;
1368               m_key.fib_index = local->fib_index;
1369               kv.key = m_key.as_u64;
1370               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1371                 {
1372                   nat_log_err ("static_mapping_by_local key del failed");
1373                   return VNET_API_ERROR_UNSPECIFIED;
1374                 }
1375             }
1376
1377           if (sm->num_workers > 1)
1378             {
1379               ip4_header_t ip = {
1380                 .src_address = local->addr,
1381               };
1382               tsm = vec_elt_at_index (sm->per_thread_data,
1383                                       sm->worker_in2out_cb (&ip, m->fib_index));
1384             }
1385           else
1386             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1387
1388           /* Delete sessions */
1389           u_key.addr = local->addr;
1390           u_key.fib_index = local->fib_index;
1391           kv.key = u_key.as_u64;
1392           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1393             {
1394               u = pool_elt_at_index (tsm->users, value.value);
1395               if (u->nstaticsessions)
1396                 {
1397                   head_index = u->sessions_per_user_list_head_index;
1398                   head = pool_elt_at_index (tsm->list_pool, head_index);
1399                   elt_index = head->next;
1400                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1401                   ses_index = elt->value;
1402                   while (ses_index != ~0)
1403                     {
1404                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1405                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1406                       ses_index = elt->value;
1407
1408                       if (!(is_lb_session (s)))
1409                         continue;
1410
1411                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1412                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1413                         continue;
1414
1415                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1416                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1417                     }
1418                 }
1419             }
1420       }));
1421       /* *INDENT-ON* */
1422       if (m->affinity)
1423         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1424       pool_free (m->locals);
1425       vec_free (m->tag);
1426       vec_free (m->workers);
1427
1428       pool_put (sm->static_mappings, m);
1429     }
1430
1431   return 0;
1432 }
1433
1434 int
1435 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1436                                        ip4_address_t l_addr, u16 l_port,
1437                                        snat_protocol_t proto, u32 vrf_id,
1438                                        u8 probability, u8 is_add)
1439 {
1440   snat_main_t *sm = &snat_main;
1441   snat_static_mapping_t *m = 0;
1442   snat_session_key_t m_key;
1443   clib_bihash_kv_8_8_t kv, value;
1444   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1445   snat_main_per_thread_data_t *tsm;
1446   snat_user_key_t u_key;
1447   snat_user_t *u;
1448   snat_session_t *s;
1449   dlist_elt_t *head, *elt;
1450   u32 elt_index, head_index, ses_index, *locals = 0;
1451   uword *bitmap = 0;
1452   int i;
1453
1454   if (!sm->endpoint_dependent)
1455     return VNET_API_ERROR_FEATURE_DISABLED;
1456
1457   m_key.addr = e_addr;
1458   m_key.port = e_port;
1459   m_key.protocol = proto;
1460   m_key.fib_index = 0;
1461   kv.key = m_key.as_u64;
1462   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1463     m = pool_elt_at_index (sm->static_mappings, value.value);
1464
1465   if (!m)
1466     return VNET_API_ERROR_NO_SUCH_ENTRY;
1467
1468   if (!is_lb_static_mapping (m))
1469     return VNET_API_ERROR_INVALID_VALUE;
1470
1471   /* *INDENT-OFF* */
1472   pool_foreach (local, m->locals,
1473   ({
1474     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1475         (local->vrf_id == vrf_id))
1476       {
1477         match_local = local;
1478         break;
1479       }
1480   }));
1481   /* *INDENT-ON* */
1482
1483   if (is_add)
1484     {
1485       if (match_local)
1486         return VNET_API_ERROR_VALUE_EXIST;
1487
1488       pool_get (m->locals, local);
1489       clib_memset (local, 0, sizeof (*local));
1490       local->addr.as_u32 = l_addr.as_u32;
1491       local->port = l_port;
1492       local->probability = probability;
1493       local->vrf_id = vrf_id;
1494       local->fib_index =
1495         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1496                                            FIB_SOURCE_PLUGIN_LOW);
1497
1498       if (!is_out2in_only_static_mapping (m))
1499         {
1500           m_key.addr = l_addr;
1501           m_key.port = l_port;
1502           m_key.fib_index = local->fib_index;
1503           kv.key = m_key.as_u64;
1504           kv.value = m - sm->static_mappings;
1505           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1506             nat_log_err ("static_mapping_by_local key add failed");
1507         }
1508     }
1509   else
1510     {
1511       if (!match_local)
1512         return VNET_API_ERROR_NO_SUCH_ENTRY;
1513
1514       if (pool_elts (m->locals) < 3)
1515         return VNET_API_ERROR_UNSPECIFIED;
1516
1517       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1518                         FIB_SOURCE_PLUGIN_LOW);
1519
1520       if (!is_out2in_only_static_mapping (m))
1521         {
1522           m_key.addr = l_addr;
1523           m_key.port = l_port;
1524           m_key.fib_index = match_local->fib_index;
1525           kv.key = m_key.as_u64;
1526           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1527             nat_log_err ("static_mapping_by_local key del failed");
1528         }
1529
1530       if (sm->num_workers > 1)
1531         {
1532           ip4_header_t ip = {
1533             .src_address = local->addr,
1534           };
1535           tsm = vec_elt_at_index (sm->per_thread_data,
1536                                   sm->worker_in2out_cb (&ip, m->fib_index));
1537         }
1538       else
1539         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1540
1541       /* Delete sessions */
1542       u_key.addr = match_local->addr;
1543       u_key.fib_index = match_local->fib_index;
1544       kv.key = u_key.as_u64;
1545       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1546         {
1547           u = pool_elt_at_index (tsm->users, value.value);
1548           if (u->nstaticsessions)
1549             {
1550               head_index = u->sessions_per_user_list_head_index;
1551               head = pool_elt_at_index (tsm->list_pool, head_index);
1552               elt_index = head->next;
1553               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1554               ses_index = elt->value;
1555               while (ses_index != ~0)
1556                 {
1557                   s = pool_elt_at_index (tsm->sessions, ses_index);
1558                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1559                   ses_index = elt->value;
1560
1561                   if (!(is_lb_session (s)))
1562                     continue;
1563
1564                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1565                       (clib_net_to_host_u16 (s->in2out.port) !=
1566                        match_local->port))
1567                     continue;
1568
1569                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1570                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1571                 }
1572             }
1573         }
1574
1575       pool_put (m->locals, match_local);
1576     }
1577
1578   vec_free (m->workers);
1579
1580   /* *INDENT-OFF* */
1581   pool_foreach (local, m->locals,
1582   ({
1583     vec_add1 (locals, local - m->locals);
1584     if (sm->num_workers > 1)
1585       {
1586         ip4_header_t ip;
1587         ip.src_address.as_u32 = local->addr.as_u32,
1588         bitmap = clib_bitmap_set (bitmap,
1589                                   sm->worker_in2out_cb (&ip, local->fib_index),
1590                                   1);
1591       }
1592   }));
1593   /* *INDENT-ON* */
1594
1595   ASSERT (vec_len (locals) > 1);
1596
1597   local = pool_elt_at_index (m->locals, locals[0]);
1598   local->prefix = local->probability;
1599   for (i = 1; i < vec_len (locals); i++)
1600     {
1601       local = pool_elt_at_index (m->locals, locals[i]);
1602       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1603       local->prefix = local->probability + prev_local->prefix;
1604     }
1605
1606   /* Assign workers */
1607   if (sm->num_workers > 1)
1608     {
1609       /* *INDENT-OFF* */
1610       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1611       /* *INDENT-ON* */
1612     }
1613
1614   return 0;
1615 }
1616
1617 int
1618 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1619                   u8 twice_nat)
1620 {
1621   snat_address_t *a = 0;
1622   snat_session_t *ses;
1623   u32 *ses_to_be_removed = 0, *ses_index;
1624   snat_main_per_thread_data_t *tsm;
1625   snat_static_mapping_t *m;
1626   snat_interface_t *interface;
1627   int i;
1628   snat_address_t *addresses =
1629     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1630
1631   /* Find SNAT address */
1632   for (i = 0; i < vec_len (addresses); i++)
1633     {
1634       if (addresses[i].addr.as_u32 == addr.as_u32)
1635         {
1636           a = addresses + i;
1637           break;
1638         }
1639     }
1640   if (!a)
1641     return VNET_API_ERROR_NO_SUCH_ENTRY;
1642
1643   if (delete_sm)
1644     {
1645       /* *INDENT-OFF* */
1646       pool_foreach (m, sm->static_mappings,
1647       ({
1648           if (m->external_addr.as_u32 == addr.as_u32)
1649             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1650                                             m->local_port, m->external_port,
1651                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1652                                             m->proto, 0, m->twice_nat,
1653                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1654       }));
1655       /* *INDENT-ON* */
1656     }
1657   else
1658     {
1659       /* Check if address is used in some static mapping */
1660       if (is_snat_address_used_in_static_mapping (sm, addr))
1661         {
1662           nat_log_notice ("address used in static mapping");
1663           return VNET_API_ERROR_UNSPECIFIED;
1664         }
1665     }
1666
1667   if (a->fib_index != ~0)
1668     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1669
1670   /* Delete sessions using address */
1671   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1672     {
1673       /* *INDENT-OFF* */
1674       vec_foreach (tsm, sm->per_thread_data)
1675         {
1676           pool_foreach (ses, tsm->sessions, ({
1677             if (ses->out2in.addr.as_u32 == addr.as_u32)
1678               {
1679                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1680                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1681               }
1682           }));
1683
1684           vec_foreach (ses_index, ses_to_be_removed)
1685             {
1686               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1687               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1688             }
1689
1690           vec_free (ses_to_be_removed);
1691         }
1692       /* *INDENT-ON* */
1693     }
1694
1695 #define _(N, i, n, s) \
1696   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1697   vec_free (a->busy_##n##_ports_per_thread);
1698   foreach_snat_protocol
1699 #undef _
1700     if (twice_nat)
1701     {
1702       vec_del1 (sm->twice_nat_addresses, i);
1703       return 0;
1704     }
1705   else
1706     vec_del1 (sm->addresses, i);
1707
1708   /* Delete external address from FIB */
1709   /* *INDENT-OFF* */
1710   pool_foreach (interface, sm->interfaces,
1711   ({
1712     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1713       continue;
1714
1715     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1716     break;
1717   }));
1718   pool_foreach (interface, sm->output_feature_interfaces,
1719   ({
1720     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1721       continue;
1722
1723     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1724     break;
1725   }));
1726   /* *INDENT-ON* */
1727
1728   return 0;
1729 }
1730
1731 int
1732 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1733 {
1734   snat_main_t *sm = &snat_main;
1735   snat_interface_t *i;
1736   const char *feature_name, *del_feature_name;
1737   snat_address_t *ap;
1738   snat_static_mapping_t *m;
1739   snat_det_map_t *dm;
1740   nat_outside_fib_t *outside_fib;
1741   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1742                                                        sw_if_index);
1743
1744   if (sm->out2in_dpo && !is_inside)
1745     return VNET_API_ERROR_UNSUPPORTED;
1746
1747   /* *INDENT-OFF* */
1748   pool_foreach (i, sm->output_feature_interfaces,
1749   ({
1750     if (i->sw_if_index == sw_if_index)
1751       return VNET_API_ERROR_VALUE_EXIST;
1752   }));
1753   /* *INDENT-ON* */
1754
1755   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1756     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1757   else
1758     {
1759       if (sm->num_workers > 1 && !sm->deterministic)
1760         feature_name =
1761           is_inside ? "nat44-in2out-worker-handoff" :
1762           "nat44-out2in-worker-handoff";
1763       else if (sm->deterministic)
1764         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1765       else if (sm->endpoint_dependent)
1766         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1767       else
1768         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1769     }
1770
1771   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1772     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1773                                                       NAT_FQ_NELTS);
1774
1775   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1776     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1777                                                       NAT_FQ_NELTS);
1778
1779   if (!is_inside)
1780     {
1781       /* *INDENT-OFF* */
1782       vec_foreach (outside_fib, sm->outside_fibs)
1783         {
1784           if (outside_fib->fib_index == fib_index)
1785             {
1786               if (is_del)
1787                 {
1788                   outside_fib->refcount--;
1789                   if (!outside_fib->refcount)
1790                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1791                 }
1792               else
1793                 outside_fib->refcount++;
1794               goto feature_set;
1795             }
1796         }
1797       /* *INDENT-ON* */
1798       if (!is_del)
1799         {
1800           vec_add2 (sm->outside_fibs, outside_fib, 1);
1801           outside_fib->refcount = 1;
1802           outside_fib->fib_index = fib_index;
1803         }
1804     }
1805 feature_set:
1806   /* *INDENT-OFF* */
1807   pool_foreach (i, sm->interfaces,
1808   ({
1809     if (i->sw_if_index == sw_if_index)
1810       {
1811         if (is_del)
1812           {
1813             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1814               {
1815                 if (is_inside)
1816                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1817                 else
1818                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1819
1820                 if (sm->num_workers > 1 && !sm->deterministic)
1821                   {
1822                     del_feature_name = "nat44-handoff-classify";
1823                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1824                                                  "nat44-out2in-worker-handoff";
1825                   }
1826                 else if (sm->deterministic)
1827                   {
1828                     del_feature_name = "nat44-det-classify";
1829                     feature_name = !is_inside ?  "nat44-det-in2out" :
1830                                                  "nat44-det-out2in";
1831                   }
1832                 else if (sm->endpoint_dependent)
1833                   {
1834                     del_feature_name = "nat44-ed-classify";
1835                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1836                                                  "nat44-ed-out2in";
1837                   }
1838                 else
1839                   {
1840                     del_feature_name = "nat44-classify";
1841                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1842                   }
1843
1844                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1845                                              sw_if_index, 0, 0, 0);
1846                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1847                                              sw_if_index, 1, 0, 0);
1848                 if (!is_inside)
1849                   {
1850                     if (sm->endpoint_dependent)
1851                       vnet_feature_enable_disable ("ip4-local",
1852                                                    "nat44-ed-hairpinning",
1853                                                    sw_if_index, 1, 0, 0);
1854                     else if (!sm->deterministic)
1855                       vnet_feature_enable_disable ("ip4-local",
1856                                                    "nat44-hairpinning",
1857                                                    sw_if_index, 1, 0, 0);
1858                   }
1859               }
1860             else
1861               {
1862                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1863                                              sw_if_index, 0, 0, 0);
1864                 pool_put (sm->interfaces, i);
1865                 if (is_inside)
1866                   {
1867                     if (sm->endpoint_dependent)
1868                       vnet_feature_enable_disable ("ip4-local",
1869                                                    "nat44-ed-hairpinning",
1870                                                    sw_if_index, 0, 0, 0);
1871                     else if (!sm->deterministic)
1872                       vnet_feature_enable_disable ("ip4-local",
1873                                                    "nat44-hairpinning",
1874                                                    sw_if_index, 0, 0, 0);
1875                   }
1876               }
1877           }
1878         else
1879           {
1880             if ((nat_interface_is_inside(i) && is_inside) ||
1881                 (nat_interface_is_outside(i) && !is_inside))
1882               return 0;
1883
1884             if (sm->num_workers > 1 && !sm->deterministic)
1885               {
1886                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1887                                                  "nat44-out2in-worker-handoff";
1888                 feature_name = "nat44-handoff-classify";
1889               }
1890             else if (sm->deterministic)
1891               {
1892                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1893                                                  "nat44-det-out2in";
1894                 feature_name = "nat44-det-classify";
1895               }
1896             else if (sm->endpoint_dependent)
1897               {
1898                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1899                                                  "nat44-ed-out2in";
1900                 feature_name = "nat44-ed-classify";
1901               }
1902             else
1903               {
1904                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1905                 feature_name = "nat44-classify";
1906               }
1907
1908             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1909                                          sw_if_index, 0, 0, 0);
1910             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1911                                          sw_if_index, 1, 0, 0);
1912             if (!is_inside)
1913               {
1914                 if (sm->endpoint_dependent)
1915                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1916                                                sw_if_index, 0, 0, 0);
1917                 else if (!sm->deterministic)
1918                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1919                                                sw_if_index, 0, 0, 0);
1920               }
1921             goto set_flags;
1922           }
1923
1924         goto fib;
1925       }
1926   }));
1927   /* *INDENT-ON* */
1928
1929   if (is_del)
1930     return VNET_API_ERROR_NO_SUCH_ENTRY;
1931
1932   pool_get (sm->interfaces, i);
1933   i->sw_if_index = sw_if_index;
1934   i->flags = 0;
1935   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1936                                0);
1937
1938   if (is_inside && !sm->out2in_dpo)
1939     {
1940       if (sm->endpoint_dependent)
1941         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1942                                      sw_if_index, 1, 0, 0);
1943       else if (!sm->deterministic)
1944         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1945                                      sw_if_index, 1, 0, 0);
1946     }
1947
1948 set_flags:
1949   if (is_inside)
1950     {
1951       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1952       return 0;
1953     }
1954   else
1955     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1956
1957   /* Add/delete external addresses to FIB */
1958 fib:
1959   /* *INDENT-OFF* */
1960   vec_foreach (ap, sm->addresses)
1961     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1962
1963   pool_foreach (m, sm->static_mappings,
1964   ({
1965     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1966       continue;
1967
1968     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1969   }));
1970
1971   pool_foreach (dm, sm->det_maps,
1972   ({
1973     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1974   }));
1975   /* *INDENT-ON* */
1976
1977   return 0;
1978 }
1979
1980 int
1981 snat_interface_add_del_output_feature (u32 sw_if_index,
1982                                        u8 is_inside, int is_del)
1983 {
1984   snat_main_t *sm = &snat_main;
1985   snat_interface_t *i;
1986   snat_address_t *ap;
1987   snat_static_mapping_t *m;
1988   nat_outside_fib_t *outside_fib;
1989   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1990                                                        sw_if_index);
1991
1992
1993   if (sm->deterministic ||
1994       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1995     return VNET_API_ERROR_UNSUPPORTED;
1996
1997   /* *INDENT-OFF* */
1998   pool_foreach (i, sm->interfaces,
1999   ({
2000     if (i->sw_if_index == sw_if_index)
2001       return VNET_API_ERROR_VALUE_EXIST;
2002   }));
2003   /* *INDENT-ON* */
2004
2005   if (!is_inside)
2006     {
2007       /* *INDENT-OFF* */
2008       vec_foreach (outside_fib, sm->outside_fibs)
2009         {
2010           if (outside_fib->fib_index == fib_index)
2011             {
2012               if (is_del)
2013                 {
2014                   outside_fib->refcount--;
2015                   if (!outside_fib->refcount)
2016                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2017                 }
2018               else
2019                 outside_fib->refcount++;
2020               goto feature_set;
2021             }
2022         }
2023       /* *INDENT-ON* */
2024       if (!is_del)
2025         {
2026           vec_add2 (sm->outside_fibs, outside_fib, 1);
2027           outside_fib->refcount = 1;
2028           outside_fib->fib_index = fib_index;
2029         }
2030     }
2031
2032 feature_set:
2033   if (is_inside)
2034     {
2035       if (sm->endpoint_dependent)
2036         {
2037           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2038                                        sw_if_index, !is_del, 0, 0);
2039           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2040                                        sw_if_index, !is_del, 0, 0);
2041         }
2042       else
2043         {
2044           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2045                                        sw_if_index, !is_del, 0, 0);
2046           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2047                                        sw_if_index, !is_del, 0, 0);
2048         }
2049       goto fq;
2050     }
2051
2052   if (sm->num_workers > 1)
2053     {
2054       vnet_feature_enable_disable ("ip4-unicast",
2055                                    "nat44-out2in-worker-handoff",
2056                                    sw_if_index, !is_del, 0, 0);
2057       vnet_feature_enable_disable ("ip4-output",
2058                                    "nat44-in2out-output-worker-handoff",
2059                                    sw_if_index, !is_del, 0, 0);
2060     }
2061   else
2062     {
2063       if (sm->endpoint_dependent)
2064         {
2065           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2066                                        sw_if_index, !is_del, 0, 0);
2067           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2068                                        sw_if_index, !is_del, 0, 0);
2069         }
2070       else
2071         {
2072           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2073                                        sw_if_index, !is_del, 0, 0);
2074           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2075                                        sw_if_index, !is_del, 0, 0);
2076         }
2077     }
2078
2079 fq:
2080   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2081     sm->fq_in2out_output_index =
2082       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2083
2084   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2085     sm->fq_out2in_index =
2086       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2087
2088   /* *INDENT-OFF* */
2089   pool_foreach (i, sm->output_feature_interfaces,
2090   ({
2091     if (i->sw_if_index == sw_if_index)
2092       {
2093         if (is_del)
2094           pool_put (sm->output_feature_interfaces, i);
2095         else
2096           return VNET_API_ERROR_VALUE_EXIST;
2097
2098         goto fib;
2099       }
2100   }));
2101   /* *INDENT-ON* */
2102
2103   if (is_del)
2104     return VNET_API_ERROR_NO_SUCH_ENTRY;
2105
2106   pool_get (sm->output_feature_interfaces, i);
2107   i->sw_if_index = sw_if_index;
2108   i->flags = 0;
2109   if (is_inside)
2110     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2111   else
2112     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2113
2114   /* Add/delete external addresses to FIB */
2115 fib:
2116   if (is_inside)
2117     return 0;
2118
2119   /* *INDENT-OFF* */
2120   vec_foreach (ap, sm->addresses)
2121     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2122
2123   pool_foreach (m, sm->static_mappings,
2124   ({
2125     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2126       continue;
2127
2128     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2129   }));
2130   /* *INDENT-ON* */
2131
2132   return 0;
2133 }
2134
2135 int
2136 snat_set_workers (uword * bitmap)
2137 {
2138   snat_main_t *sm = &snat_main;
2139   int i, j = 0;
2140
2141   if (sm->num_workers < 2)
2142     return VNET_API_ERROR_FEATURE_DISABLED;
2143
2144   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2145     return VNET_API_ERROR_INVALID_WORKER;
2146
2147   vec_free (sm->workers);
2148   /* *INDENT-OFF* */
2149   clib_bitmap_foreach (i, bitmap,
2150     ({
2151       vec_add1(sm->workers, i);
2152       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2153       j++;
2154     }));
2155   /* *INDENT-ON* */
2156
2157   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2158   sm->num_snat_thread = _vec_len (sm->workers);
2159
2160   return 0;
2161 }
2162
2163 static void
2164 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2165                          u32 old_fib_index)
2166 {
2167   snat_main_t *sm = &snat_main;
2168   nat_outside_fib_t *outside_fib;
2169   snat_interface_t *i;
2170   u8 is_add = 1;
2171
2172   if (new_fib_index == old_fib_index)
2173     return;
2174
2175   if (!vec_len (sm->outside_fibs))
2176     return;
2177
2178   pool_foreach (i, sm->interfaces, (
2179                                      {
2180                                      if (i->sw_if_index == sw_if_index)
2181                                      {
2182                                      if (!(nat_interface_is_outside (i)))
2183                                      return;}
2184                                      }
2185                 ));
2186   vec_foreach (outside_fib, sm->outside_fibs)
2187   {
2188     if (outside_fib->fib_index == old_fib_index)
2189       {
2190         outside_fib->refcount--;
2191         if (!outside_fib->refcount)
2192           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2193         break;
2194       }
2195   }
2196
2197   vec_foreach (outside_fib, sm->outside_fibs)
2198   {
2199     if (outside_fib->fib_index == new_fib_index)
2200       {
2201         outside_fib->refcount++;
2202         is_add = 0;
2203         break;
2204       }
2205   }
2206
2207   if (is_add)
2208     {
2209       vec_add2 (sm->outside_fibs, outside_fib, 1);
2210       outside_fib->refcount = 1;
2211       outside_fib->fib_index = new_fib_index;
2212     }
2213 }
2214
2215 static void
2216 snat_ip4_table_bind (ip4_main_t * im,
2217                      uword opaque,
2218                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2219 {
2220   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2221 }
2222
2223 static void
2224 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2225                                        uword opaque,
2226                                        u32 sw_if_index,
2227                                        ip4_address_t * address,
2228                                        u32 address_length,
2229                                        u32 if_address_index, u32 is_delete);
2230
2231 static void
2232 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2233                                  uword opaque,
2234                                  u32 sw_if_index,
2235                                  ip4_address_t * address,
2236                                  u32 address_length,
2237                                  u32 if_address_index, u32 is_delete);
2238
2239 static int
2240 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2241                                  u32 fib_index,
2242                                  u32 thread_index,
2243                                  snat_session_key_t * k,
2244                                  u16 port_per_thread, u32 snat_thread_index);
2245
2246 static clib_error_t *
2247 snat_init (vlib_main_t * vm)
2248 {
2249   snat_main_t *sm = &snat_main;
2250   clib_error_t *error = 0;
2251   ip4_main_t *im = &ip4_main;
2252   ip_lookup_main_t *lm = &im->lookup_main;
2253   uword *p;
2254   vlib_thread_registration_t *tr;
2255   vlib_thread_main_t *tm = vlib_get_thread_main ();
2256   uword *bitmap = 0;
2257   u32 i;
2258   ip4_add_del_interface_address_callback_t cb4;
2259   vlib_node_t *node;
2260
2261   sm->vlib_main = vm;
2262   sm->vnet_main = vnet_get_main ();
2263   sm->ip4_main = im;
2264   sm->ip4_lookup_main = lm;
2265   sm->api_main = &api_main;
2266   sm->first_worker_index = 0;
2267   sm->num_workers = 0;
2268   sm->num_snat_thread = 1;
2269   sm->workers = 0;
2270   sm->port_per_thread = 0xffff - 1024;
2271   sm->fq_in2out_index = ~0;
2272   sm->fq_in2out_output_index = ~0;
2273   sm->fq_out2in_index = ~0;
2274   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2275   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2276   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2277   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2278   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2279   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2280   sm->forwarding_enabled = 0;
2281   sm->log_class = vlib_log_register_class ("nat", 0);
2282   sm->mss_clamping = 0;
2283
2284   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2285   sm->error_node_index = node->index;
2286
2287   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2288   sm->in2out_node_index = node->index;
2289   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2290   sm->in2out_output_node_index = node->index;
2291   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2292   sm->in2out_fast_node_index = node->index;
2293   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2294   sm->in2out_slowpath_node_index = node->index;
2295   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2296   sm->in2out_slowpath_output_node_index = node->index;
2297   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-reass");
2298   sm->in2out_reass_node_index = node->index;
2299
2300   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2301   sm->ed_in2out_node_index = node->index;
2302   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2303   sm->ed_in2out_slowpath_node_index = node->index;
2304   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-reass");
2305   sm->ed_in2out_reass_node_index = node->index;
2306
2307   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2308   sm->out2in_node_index = node->index;
2309   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2310   sm->out2in_fast_node_index = node->index;
2311   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-reass");
2312   sm->out2in_reass_node_index = node->index;
2313
2314   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2315   sm->ed_out2in_node_index = node->index;
2316   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2317   sm->ed_out2in_slowpath_node_index = node->index;
2318   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-reass");
2319   sm->ed_out2in_reass_node_index = node->index;
2320
2321   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2322   sm->det_in2out_node_index = node->index;
2323   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2324   sm->det_out2in_node_index = node->index;
2325
2326   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2327   sm->hairpinning_node_index = node->index;
2328   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2329   sm->hairpin_dst_node_index = node->index;
2330   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2331   sm->hairpin_src_node_index = node->index;
2332   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2333   sm->ed_hairpinning_node_index = node->index;
2334   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2335   sm->ed_hairpin_dst_node_index = node->index;
2336   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2337   sm->ed_hairpin_src_node_index = node->index;
2338
2339   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2340   if (p)
2341     {
2342       tr = (vlib_thread_registration_t *) p[0];
2343       if (tr)
2344         {
2345           sm->num_workers = tr->count;
2346           sm->first_worker_index = tr->first_index;
2347         }
2348     }
2349
2350   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2351
2352   /* Use all available workers by default */
2353   if (sm->num_workers > 1)
2354     {
2355       for (i = 0; i < sm->num_workers; i++)
2356         bitmap = clib_bitmap_set (bitmap, i, 1);
2357       snat_set_workers (bitmap);
2358       clib_bitmap_free (bitmap);
2359     }
2360   else
2361     {
2362       sm->per_thread_data[0].snat_thread_index = 0;
2363     }
2364
2365   error = snat_api_init (vm, sm);
2366   if (error)
2367     return error;
2368
2369   /* Set up the interface address add/del callback */
2370   cb4.function = snat_ip4_add_del_interface_address_cb;
2371   cb4.function_opaque = 0;
2372
2373   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2374
2375   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2376   cb4.function_opaque = 0;
2377
2378   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2379
2380   nat_dpo_module_init ();
2381
2382   /* Init counters */
2383   sm->total_users.name = "total-users";
2384   sm->total_users.stat_segment_name = "/nat44/total-users";
2385   vlib_validate_simple_counter (&sm->total_users, 0);
2386   vlib_zero_simple_counter (&sm->total_users, 0);
2387   sm->total_sessions.name = "total-sessions";
2388   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2389   vlib_validate_simple_counter (&sm->total_sessions, 0);
2390   vlib_zero_simple_counter (&sm->total_sessions, 0);
2391
2392   /* Init IPFIX logging */
2393   snat_ipfix_logging_init (vm);
2394
2395   /* Init NAT64 */
2396   error = nat64_init (vm);
2397   if (error)
2398     return error;
2399
2400   dslite_init (vm);
2401
2402   nat66_init (vm);
2403
2404   ip4_table_bind_callback_t cbt4 = {
2405     .function = snat_ip4_table_bind,
2406   };
2407   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2408
2409   /* Init virtual fragmenentation reassembly */
2410   return nat_reass_init (vm);
2411 }
2412
2413 VLIB_INIT_FUNCTION (snat_init);
2414
2415 void
2416 snat_free_outside_address_and_port (snat_address_t * addresses,
2417                                     u32 thread_index, snat_session_key_t * k)
2418 {
2419   snat_address_t *a;
2420   u32 address_index;
2421   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2422
2423   for (address_index = 0; address_index < vec_len (addresses);
2424        address_index++)
2425     {
2426       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2427         break;
2428     }
2429
2430   ASSERT (address_index < vec_len (addresses));
2431
2432   a = addresses + address_index;
2433
2434   switch (k->protocol)
2435     {
2436 #define _(N, i, n, s) \
2437     case SNAT_PROTOCOL_##N: \
2438       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2439         port_host_byte_order) == 1); \
2440       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2441         port_host_byte_order, 0); \
2442       a->busy_##n##_ports--; \
2443       a->busy_##n##_ports_per_thread[thread_index]--; \
2444       break;
2445       foreach_snat_protocol
2446 #undef _
2447     default:
2448       nat_log_info ("unknown protocol");
2449       return;
2450     }
2451 }
2452
2453 static int
2454 nat_set_outside_address_and_port (snat_address_t * addresses,
2455                                   u32 thread_index, snat_session_key_t * k)
2456 {
2457   snat_address_t *a = 0;
2458   u32 address_index;
2459   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2460
2461   for (address_index = 0; address_index < vec_len (addresses);
2462        address_index++)
2463     {
2464       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2465         continue;
2466
2467       a = addresses + address_index;
2468       switch (k->protocol)
2469         {
2470 #define _(N, j, n, s) \
2471         case SNAT_PROTOCOL_##N: \
2472           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2473             return VNET_API_ERROR_INSTANCE_IN_USE; \
2474           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2475           a->busy_##n##_ports_per_thread[thread_index]++; \
2476           a->busy_##n##_ports++; \
2477           return 0;
2478           foreach_snat_protocol
2479 #undef _
2480         default:
2481           nat_log_info ("unknown protocol");
2482           return 1;
2483         }
2484     }
2485
2486   return VNET_API_ERROR_NO_SUCH_ENTRY;
2487 }
2488
2489 int
2490 snat_static_mapping_match (snat_main_t * sm,
2491                            snat_session_key_t match,
2492                            snat_session_key_t * mapping,
2493                            u8 by_external,
2494                            u8 * is_addr_only,
2495                            twice_nat_type_t * twice_nat,
2496                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2497                            u8 * is_identity_nat)
2498 {
2499   clib_bihash_kv_8_8_t kv, value;
2500   snat_static_mapping_t *m;
2501   snat_session_key_t m_key;
2502   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2503   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2504   u8 backend_index;
2505   nat44_lb_addr_port_t *local;
2506
2507   m_key.fib_index = match.fib_index;
2508   if (by_external)
2509     {
2510       mapping_hash = &sm->static_mapping_by_external;
2511       m_key.fib_index = 0;
2512     }
2513
2514   m_key.addr = match.addr;
2515   m_key.port = clib_net_to_host_u16 (match.port);
2516   m_key.protocol = match.protocol;
2517
2518   kv.key = m_key.as_u64;
2519
2520   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2521     {
2522       /* Try address only mapping */
2523       m_key.port = 0;
2524       m_key.protocol = 0;
2525       kv.key = m_key.as_u64;
2526       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2527         return 1;
2528     }
2529
2530   m = pool_elt_at_index (sm->static_mappings, value.value);
2531
2532   if (by_external)
2533     {
2534       if (is_lb_static_mapping (m))
2535         {
2536           if (PREDICT_FALSE (lb != 0))
2537             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2538           if (m->affinity)
2539             {
2540               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2541                                               match.protocol, match.port,
2542                                               &backend_index))
2543                 goto get_local;
2544
2545               local = pool_elt_at_index (m->locals, backend_index);
2546               mapping->addr = local->addr;
2547               mapping->port = clib_host_to_net_u16 (local->port);
2548               mapping->fib_index = local->fib_index;
2549               goto end;
2550             }
2551         get_local:
2552           /* *INDENT-OFF* */
2553           pool_foreach_index (i, m->locals,
2554           ({
2555             vec_add1 (tmp, i);
2556           }));
2557           /* *INDENT-ON* */
2558           hi = vec_len (tmp) - 1;
2559           local = pool_elt_at_index (m->locals, tmp[hi]);
2560           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2561           while (lo < hi)
2562             {
2563               mid = ((hi - lo) >> 1) + lo;
2564               local = pool_elt_at_index (m->locals, tmp[mid]);
2565               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2566             }
2567           local = pool_elt_at_index (m->locals, tmp[lo]);
2568           if (!(local->prefix >= rand))
2569             return 1;
2570           if (PREDICT_FALSE (sm->num_workers > 1))
2571             {
2572               ip4_header_t ip = {
2573                 .src_address = local->addr,
2574               };
2575               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2576                   vlib_get_thread_index ())
2577                 goto get_local;
2578             }
2579           mapping->addr = local->addr;
2580           mapping->port = clib_host_to_net_u16 (local->port);
2581           mapping->fib_index = local->fib_index;
2582           if (m->affinity)
2583             {
2584               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2585                                                 match.protocol, match.port,
2586                                                 tmp[lo], m->affinity,
2587                                                 m->affinity_per_service_list_head_index))
2588                 nat_log_info ("create affinity record failed");
2589             }
2590           vec_free (tmp);
2591         }
2592       else
2593         {
2594           if (PREDICT_FALSE (lb != 0))
2595             *lb = NO_LB_NAT;
2596           mapping->fib_index = m->fib_index;
2597           mapping->addr = m->local_addr;
2598           /* Address only mapping doesn't change port */
2599           mapping->port = is_addr_only_static_mapping (m) ? match.port
2600             : clib_host_to_net_u16 (m->local_port);
2601         }
2602       mapping->protocol = m->proto;
2603     }
2604   else
2605     {
2606       mapping->addr = m->external_addr;
2607       /* Address only mapping doesn't change port */
2608       mapping->port = is_addr_only_static_mapping (m) ? match.port
2609         : clib_host_to_net_u16 (m->external_port);
2610       mapping->fib_index = sm->outside_fib_index;
2611     }
2612
2613 end:
2614   if (PREDICT_FALSE (is_addr_only != 0))
2615     *is_addr_only = is_addr_only_static_mapping (m);
2616
2617   if (PREDICT_FALSE (twice_nat != 0))
2618     *twice_nat = m->twice_nat;
2619
2620   if (PREDICT_FALSE (is_identity_nat != 0))
2621     *is_identity_nat = is_identity_static_mapping (m);
2622
2623   return 0;
2624 }
2625
2626 static_always_inline u16
2627 snat_random_port (u16 min, u16 max)
2628 {
2629   snat_main_t *sm = &snat_main;
2630   return min + random_u32 (&sm->random_seed) /
2631     (random_u32_max () / (max - min + 1) + 1);
2632 }
2633
2634 int
2635 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2636                                      u32 fib_index,
2637                                      u32 thread_index,
2638                                      snat_session_key_t * k,
2639                                      u16 port_per_thread,
2640                                      u32 snat_thread_index)
2641 {
2642   snat_main_t *sm = &snat_main;
2643
2644   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2645                                   port_per_thread, snat_thread_index);
2646 }
2647
2648 static int
2649 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2650                                  u32 fib_index,
2651                                  u32 thread_index,
2652                                  snat_session_key_t * k,
2653                                  u16 port_per_thread, u32 snat_thread_index)
2654 {
2655   int i;
2656   snat_address_t *a, *ga = 0;
2657   u32 portnum;
2658
2659   for (i = 0; i < vec_len (addresses); i++)
2660     {
2661       a = addresses + i;
2662       switch (k->protocol)
2663         {
2664 #define _(N, j, n, s) \
2665         case SNAT_PROTOCOL_##N: \
2666           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2667             { \
2668               if (a->fib_index == fib_index) \
2669                 { \
2670                   while (1) \
2671                     { \
2672                       portnum = (port_per_thread * \
2673                         snat_thread_index) + \
2674                         snat_random_port(1, port_per_thread) + 1024; \
2675                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2676                         continue; \
2677                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2678                       a->busy_##n##_ports_per_thread[thread_index]++; \
2679                       a->busy_##n##_ports++; \
2680                       k->addr = a->addr; \
2681                       k->port = clib_host_to_net_u16(portnum); \
2682                       return 0; \
2683                     } \
2684                 } \
2685               else if (a->fib_index == ~0) \
2686                 { \
2687                   ga = a; \
2688                 } \
2689             } \
2690           break;
2691           foreach_snat_protocol
2692 #undef _
2693         default:
2694           nat_log_info ("unknown protocol");
2695           return 1;
2696         }
2697
2698     }
2699
2700   if (ga)
2701     {
2702       a = ga;
2703       switch (k->protocol)
2704         {
2705 #define _(N, j, n, s) \
2706         case SNAT_PROTOCOL_##N: \
2707           while (1) \
2708             { \
2709               portnum = (port_per_thread * \
2710                 snat_thread_index) + \
2711                 snat_random_port(1, port_per_thread) + 1024; \
2712               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2713                 continue; \
2714               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2715               a->busy_##n##_ports_per_thread[thread_index]++; \
2716               a->busy_##n##_ports++; \
2717               k->addr = a->addr; \
2718               k->port = clib_host_to_net_u16(portnum); \
2719               return 0; \
2720             }
2721           break;
2722           foreach_snat_protocol
2723 #undef _
2724         default:
2725           nat_log_info ("unknown protocol");
2726           return 1;
2727         }
2728     }
2729
2730   /* Totally out of translations to use... */
2731   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2732   return 1;
2733 }
2734
2735 static int
2736 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2737                               u32 fib_index,
2738                               u32 thread_index,
2739                               snat_session_key_t * k,
2740                               u16 port_per_thread, u32 snat_thread_index)
2741 {
2742   snat_main_t *sm = &snat_main;
2743   snat_address_t *a = addresses;
2744   u16 m, ports, portnum, A, j;
2745   m = 16 - (sm->psid_offset + sm->psid_length);
2746   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2747
2748   if (!vec_len (addresses))
2749     goto exhausted;
2750
2751   switch (k->protocol)
2752     {
2753 #define _(N, i, n, s) \
2754     case SNAT_PROTOCOL_##N: \
2755       if (a->busy_##n##_ports < ports) \
2756         { \
2757           while (1) \
2758             { \
2759               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2760               j = snat_random_port(0, pow2_mask(m)); \
2761               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2762               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2763                 continue; \
2764               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2765               a->busy_##n##_ports++; \
2766               k->addr = a->addr; \
2767               k->port = clib_host_to_net_u16 (portnum); \
2768               return 0; \
2769             } \
2770         } \
2771       break;
2772       foreach_snat_protocol
2773 #undef _
2774     default:
2775       nat_log_info ("unknown protocol");
2776       return 1;
2777     }
2778
2779 exhausted:
2780   /* Totally out of translations to use... */
2781   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2782   return 1;
2783 }
2784
2785 static int
2786 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2787                                u32 fib_index,
2788                                u32 thread_index,
2789                                snat_session_key_t * k,
2790                                u16 port_per_thread, u32 snat_thread_index)
2791 {
2792   snat_main_t *sm = &snat_main;
2793   snat_address_t *a = addresses;
2794   u16 portnum, ports;
2795
2796   ports = sm->end_port - sm->start_port + 1;
2797
2798   if (!vec_len (addresses))
2799     goto exhausted;
2800
2801   switch (k->protocol)
2802     {
2803 #define _(N, i, n, s) \
2804     case SNAT_PROTOCOL_##N: \
2805       if (a->busy_##n##_ports < ports) \
2806         { \
2807           while (1) \
2808             { \
2809               portnum = snat_random_port(sm->start_port, sm->end_port); \
2810               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2811                 continue; \
2812               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2813               a->busy_##n##_ports++; \
2814               k->addr = a->addr; \
2815               k->port = clib_host_to_net_u16 (portnum); \
2816               return 0; \
2817             } \
2818         } \
2819       break;
2820       foreach_snat_protocol
2821 #undef _
2822     default:
2823       nat_log_info ("unknown protocol");
2824       return 1;
2825     }
2826
2827 exhausted:
2828   /* Totally out of translations to use... */
2829   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2830   return 1;
2831 }
2832
2833 void
2834 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2835 {
2836   dpo_id_t dpo_v4 = DPO_INVALID;
2837   fib_prefix_t pfx = {
2838     .fp_proto = FIB_PROTOCOL_IP4,
2839     .fp_len = 32,
2840     .fp_addr.ip4.as_u32 = addr.as_u32,
2841   };
2842
2843   if (is_add)
2844     {
2845       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2846       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2847                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2848       dpo_reset (&dpo_v4);
2849     }
2850   else
2851     {
2852       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2853     }
2854 }
2855
2856 u8 *
2857 format_session_kvp (u8 * s, va_list * args)
2858 {
2859   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2860   snat_session_key_t k;
2861
2862   k.as_u64 = v->key;
2863
2864   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2865
2866   return s;
2867 }
2868
2869 u8 *
2870 format_static_mapping_kvp (u8 * s, va_list * args)
2871 {
2872   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2873   snat_session_key_t k;
2874
2875   k.as_u64 = v->key;
2876
2877   s = format (s, "%U static-mapping-index %llu",
2878               format_static_mapping_key, &k, v->value);
2879
2880   return s;
2881 }
2882
2883 u8 *
2884 format_user_kvp (u8 * s, va_list * args)
2885 {
2886   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2887   snat_user_key_t k;
2888
2889   k.as_u64 = v->key;
2890
2891   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2892               k.fib_index, v->value);
2893
2894   return s;
2895 }
2896
2897 u8 *
2898 format_ed_session_kvp (u8 * s, va_list * args)
2899 {
2900   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2901   nat_ed_ses_key_t k;
2902
2903   k.as_u64[0] = v->key[0];
2904   k.as_u64[1] = v->key[1];
2905
2906   s =
2907     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2908             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2909             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2910             format_ip_protocol, k.proto, k.fib_index, v->value);
2911
2912   return s;
2913 }
2914
2915 static u32
2916 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2917 {
2918   snat_main_t *sm = &snat_main;
2919   u32 next_worker_index = 0;
2920   u32 hash;
2921
2922   next_worker_index = sm->first_worker_index;
2923   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2924     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2925
2926   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2927     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2928   else
2929     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2930
2931   return next_worker_index;
2932 }
2933
2934 static u32
2935 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2936 {
2937   snat_main_t *sm = &snat_main;
2938   udp_header_t *udp;
2939   u16 port;
2940   snat_session_key_t m_key;
2941   clib_bihash_kv_8_8_t kv, value;
2942   snat_static_mapping_t *m;
2943   u32 proto;
2944   u32 next_worker_index = 0;
2945
2946   /* first try static mappings without port */
2947   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2948     {
2949       m_key.addr = ip0->dst_address;
2950       m_key.port = 0;
2951       m_key.protocol = 0;
2952       m_key.fib_index = rx_fib_index0;
2953       kv.key = m_key.as_u64;
2954       if (!clib_bihash_search_8_8
2955           (&sm->static_mapping_by_external, &kv, &value))
2956         {
2957           m = pool_elt_at_index (sm->static_mappings, value.value);
2958           return m->workers[0];
2959         }
2960     }
2961
2962   proto = ip_proto_to_snat_proto (ip0->protocol);
2963   udp = ip4_next_header (ip0);
2964   port = udp->dst_port;
2965
2966   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2967     {
2968       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2969         return vlib_get_thread_index ();
2970
2971       nat_reass_ip4_t *reass;
2972       reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2973                                   ip0->fragment_id, ip0->protocol);
2974
2975       if (reass && (reass->thread_index != (u32) ~ 0))
2976         return reass->thread_index;
2977
2978       if (ip4_is_first_fragment (ip0))
2979         {
2980           reass =
2981             nat_ip4_reass_create (ip0->src_address, ip0->dst_address,
2982                                   ip0->fragment_id, ip0->protocol);
2983           if (!reass)
2984             goto no_reass;
2985
2986           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2987             {
2988               m_key.addr = ip0->dst_address;
2989               m_key.port = clib_net_to_host_u16 (port);
2990               m_key.protocol = proto;
2991               m_key.fib_index = rx_fib_index0;
2992               kv.key = m_key.as_u64;
2993               if (!clib_bihash_search_8_8
2994                   (&sm->static_mapping_by_external, &kv, &value))
2995                 {
2996                   m = pool_elt_at_index (sm->static_mappings, value.value);
2997                   reass->thread_index = m->workers[0];
2998                   return reass->thread_index;
2999                 }
3000             }
3001           reass->thread_index = sm->first_worker_index;
3002           reass->thread_index +=
3003             sm->workers[(clib_net_to_host_u16 (port) - 1024) /
3004                         sm->port_per_thread];
3005           return reass->thread_index;
3006         }
3007       else
3008         return vlib_get_thread_index ();
3009     }
3010
3011 no_reass:
3012   /* unknown protocol */
3013   if (PREDICT_FALSE (proto == ~0))
3014     {
3015       /* use current thread */
3016       return vlib_get_thread_index ();
3017     }
3018
3019   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3020     {
3021       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3022       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3023       if (!icmp_is_error_message (icmp))
3024         port = echo->identifier;
3025       else
3026         {
3027           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3028           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3029           void *l4_header = ip4_next_header (inner_ip);
3030           switch (proto)
3031             {
3032             case SNAT_PROTOCOL_ICMP:
3033               icmp = (icmp46_header_t *) l4_header;
3034               echo = (icmp_echo_header_t *) (icmp + 1);
3035               port = echo->identifier;
3036               break;
3037             case SNAT_PROTOCOL_UDP:
3038             case SNAT_PROTOCOL_TCP:
3039               port = ((tcp_udp_header_t *) l4_header)->src_port;
3040               break;
3041             default:
3042               return vlib_get_thread_index ();
3043             }
3044         }
3045     }
3046
3047   /* try static mappings with port */
3048   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3049     {
3050       m_key.addr = ip0->dst_address;
3051       m_key.port = clib_net_to_host_u16 (port);
3052       m_key.protocol = proto;
3053       m_key.fib_index = rx_fib_index0;
3054       kv.key = m_key.as_u64;
3055       if (!clib_bihash_search_8_8
3056           (&sm->static_mapping_by_external, &kv, &value))
3057         {
3058           m = pool_elt_at_index (sm->static_mappings, value.value);
3059           return m->workers[0];
3060         }
3061     }
3062
3063   /* worker by outside port */
3064   next_worker_index = sm->first_worker_index;
3065   next_worker_index +=
3066     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3067   return next_worker_index;
3068 }
3069
3070 static u32
3071 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
3072 {
3073   snat_main_t *sm = &snat_main;
3074   clib_bihash_kv_8_8_t kv, value;
3075   u32 proto, next_worker_index = 0;
3076   udp_header_t *udp;
3077   u16 port;
3078   snat_static_mapping_t *m;
3079   u32 hash;
3080
3081   /* first try static mappings without port */
3082   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3083     {
3084       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
3085       if (!clib_bihash_search_8_8
3086           (&sm->static_mapping_by_external, &kv, &value))
3087         {
3088           m = pool_elt_at_index (sm->static_mappings, value.value);
3089           return m->workers[0];
3090         }
3091     }
3092
3093   proto = ip_proto_to_snat_proto (ip->protocol);
3094
3095   /* unknown protocol */
3096   if (PREDICT_FALSE (proto == ~0))
3097     {
3098       /* use current thread */
3099       return vlib_get_thread_index ();
3100     }
3101
3102   udp = ip4_next_header (ip);
3103   port = udp->dst_port;
3104
3105   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3106     {
3107       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3108       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3109       if (!icmp_is_error_message (icmp))
3110         port = echo->identifier;
3111       else
3112         {
3113           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3114           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3115           void *l4_header = ip4_next_header (inner_ip);
3116           switch (proto)
3117             {
3118             case SNAT_PROTOCOL_ICMP:
3119               icmp = (icmp46_header_t *) l4_header;
3120               echo = (icmp_echo_header_t *) (icmp + 1);
3121               port = echo->identifier;
3122               break;
3123             case SNAT_PROTOCOL_UDP:
3124             case SNAT_PROTOCOL_TCP:
3125               port = ((tcp_udp_header_t *) l4_header)->src_port;
3126               break;
3127             default:
3128               return vlib_get_thread_index ();
3129             }
3130         }
3131     }
3132
3133   /* try static mappings with port */
3134   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3135     {
3136       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
3137                   clib_net_to_host_u16 (port));
3138       if (!clib_bihash_search_8_8
3139           (&sm->static_mapping_by_external, &kv, &value))
3140         {
3141           m = pool_elt_at_index (sm->static_mappings, value.value);
3142           if (!is_lb_static_mapping (m))
3143             return m->workers[0];
3144
3145           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3146             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3147
3148           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3149             return m->workers[hash & (_vec_len (m->workers) - 1)];
3150           else
3151             return m->workers[hash % _vec_len (m->workers)];
3152         }
3153     }
3154
3155   /* worker by outside port */
3156   next_worker_index = sm->first_worker_index;
3157   next_worker_index +=
3158     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3159
3160   return next_worker_index;
3161 }
3162
3163 void
3164 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3165                 ip4_address_t * out_addr, u16 out_port,
3166                 ip4_address_t * eh_addr, u16 eh_port,
3167                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3168                 u32 fib_index, u16 flags, u32 thread_index)
3169 {
3170   snat_main_t *sm = &snat_main;
3171   snat_session_key_t key;
3172   snat_user_t *u;
3173   snat_session_t *s;
3174   clib_bihash_kv_8_8_t kv;
3175   f64 now = vlib_time_now (sm->vlib_main);
3176   nat_outside_fib_t *outside_fib;
3177   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3178   snat_main_per_thread_data_t *tsm;
3179   fib_prefix_t pfx = {
3180     .fp_proto = FIB_PROTOCOL_IP4,
3181     .fp_len = 32,
3182     .fp_addr = {
3183                 .ip4.as_u32 = eh_addr->as_u32,
3184                 },
3185   };
3186
3187   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3188
3189   key.addr.as_u32 = out_addr->as_u32;
3190   key.port = out_port;
3191   key.protocol = proto;
3192
3193   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3194     {
3195       if (nat_set_outside_address_and_port
3196           (sm->addresses, thread_index, &key))
3197         return;
3198     }
3199
3200   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3201   if (!u)
3202     return;
3203
3204   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3205   if (!s)
3206     return;
3207
3208   s->last_heard = now;
3209   s->flags = flags;
3210   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3211   s->ext_host_port = eh_port;
3212   user_session_increment (sm, u, snat_is_session_static (s));
3213   switch (vec_len (sm->outside_fibs))
3214     {
3215     case 0:
3216       key.fib_index = sm->outside_fib_index;
3217       break;
3218     case 1:
3219       key.fib_index = sm->outside_fibs[0].fib_index;
3220       break;
3221     default:
3222       /* *INDENT-OFF* */
3223       vec_foreach (outside_fib, sm->outside_fibs)
3224         {
3225           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3226           if (FIB_NODE_INDEX_INVALID != fei)
3227             {
3228               if (fib_entry_get_resolving_interface (fei) != ~0)
3229                 {
3230                   key.fib_index = outside_fib->fib_index;
3231                   break;
3232                 }
3233             }
3234         }
3235       /* *INDENT-ON* */
3236       break;
3237     }
3238   s->out2in = key;
3239   kv.key = key.as_u64;
3240   kv.value = s - tsm->sessions;
3241   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3242     nat_log_warn ("out2in key add failed");
3243
3244   key.addr.as_u32 = in_addr->as_u32;
3245   key.port = in_port;
3246   key.fib_index = fib_index;
3247   s->in2out = key;
3248   kv.key = key.as_u64;
3249   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3250     nat_log_warn ("in2out key add failed");
3251 }
3252
3253 void
3254 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3255                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3256                 u32 ti)
3257 {
3258   snat_main_t *sm = &snat_main;
3259   snat_session_key_t key;
3260   clib_bihash_kv_8_8_t kv, value;
3261   u32 thread_index;
3262   snat_session_t *s;
3263   snat_main_per_thread_data_t *tsm;
3264
3265   if (sm->num_workers > 1)
3266     thread_index =
3267       sm->first_worker_index +
3268       (sm->workers[(clib_net_to_host_u16 (out_port) -
3269                     1024) / sm->port_per_thread]);
3270   else
3271     thread_index = sm->num_workers;
3272   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3273
3274   key.addr.as_u32 = out_addr->as_u32;
3275   key.port = out_port;
3276   key.protocol = proto;
3277   key.fib_index = fib_index;
3278   kv.key = key.as_u64;
3279   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3280     return;
3281
3282   s = pool_elt_at_index (tsm->sessions, value.value);
3283   nat_free_session_data (sm, s, thread_index, 1);
3284   nat44_delete_session (sm, s, thread_index);
3285 }
3286
3287 void
3288 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3289                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3290                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3291 {
3292   snat_main_t *sm = &snat_main;
3293   snat_session_key_t key;
3294   clib_bihash_kv_8_8_t kv, value;
3295   snat_session_t *s;
3296   snat_main_per_thread_data_t *tsm;
3297
3298   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3299
3300   key.addr.as_u32 = out_addr->as_u32;
3301   key.port = out_port;
3302   key.protocol = proto;
3303   key.fib_index = fib_index;
3304   kv.key = key.as_u64;
3305   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3306     return;
3307
3308   s = pool_elt_at_index (tsm->sessions, value.value);
3309   s->total_pkts = total_pkts;
3310   s->total_bytes = total_bytes;
3311 }
3312
3313 void
3314 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3315                    ip4_address_t * out_addr, u16 out_port,
3316                    ip4_address_t * eh_addr, u16 eh_port,
3317                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3318                    u32 fib_index, u16 flags, u32 thread_index)
3319 {
3320   snat_main_t *sm = &snat_main;
3321   snat_session_key_t key;
3322   snat_user_t *u;
3323   snat_session_t *s;
3324   clib_bihash_kv_16_8_t kv;
3325   f64 now = vlib_time_now (sm->vlib_main);
3326   nat_outside_fib_t *outside_fib;
3327   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3328   snat_main_per_thread_data_t *tsm;
3329   fib_prefix_t pfx = {
3330     .fp_proto = FIB_PROTOCOL_IP4,
3331     .fp_len = 32,
3332     .fp_addr = {
3333                 .ip4.as_u32 = eh_addr->as_u32,
3334                 },
3335   };
3336
3337   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3338
3339   key.addr.as_u32 = out_addr->as_u32;
3340   key.port = out_port;
3341   key.protocol = proto;
3342
3343   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3344     {
3345       if (nat_set_outside_address_and_port
3346           (sm->addresses, thread_index, &key))
3347         return;
3348     }
3349
3350   key.addr.as_u32 = ehn_addr->as_u32;
3351   key.port = ehn_port;
3352   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3353     {
3354       if (nat_set_outside_address_and_port
3355           (sm->twice_nat_addresses, thread_index, &key))
3356         return;
3357     }
3358
3359   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3360   if (!u)
3361     return;
3362
3363   s = nat_ed_session_alloc (sm, u, thread_index, now);
3364   if (!s)
3365     return;
3366
3367   s->last_heard = now;
3368   s->flags = flags;
3369   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3370   s->ext_host_nat_port = s->ext_host_port = eh_port;
3371   if (is_twice_nat_session (s))
3372     {
3373       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3374       s->ext_host_nat_port = ehn_port;
3375     }
3376   user_session_increment (sm, u, snat_is_session_static (s));
3377   switch (vec_len (sm->outside_fibs))
3378     {
3379     case 0:
3380       key.fib_index = sm->outside_fib_index;
3381       break;
3382     case 1:
3383       key.fib_index = sm->outside_fibs[0].fib_index;
3384       break;
3385     default:
3386       /* *INDENT-OFF* */
3387       vec_foreach (outside_fib, sm->outside_fibs)
3388         {
3389           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3390           if (FIB_NODE_INDEX_INVALID != fei)
3391             {
3392               if (fib_entry_get_resolving_interface (fei) != ~0)
3393                 {
3394                   key.fib_index = outside_fib->fib_index;
3395                   break;
3396                 }
3397             }
3398         }
3399       /* *INDENT-ON* */
3400       break;
3401     }
3402   key.addr.as_u32 = out_addr->as_u32;
3403   key.port = out_port;
3404   s->out2in = key;
3405   kv.value = s - tsm->sessions;
3406
3407   key.addr.as_u32 = in_addr->as_u32;
3408   key.port = in_port;
3409   key.fib_index = fib_index;
3410   s->in2out = key;
3411
3412   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3413               snat_proto_to_ip_proto (proto), fib_index, in_port,
3414               s->ext_host_nat_port);
3415   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3416     nat_log_warn ("in2out key add failed");
3417
3418   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3419               s->out2in.fib_index, out_port, eh_port);
3420   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3421     nat_log_warn ("out2in key add failed");
3422 }
3423
3424 void
3425 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3426                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3427                    u32 fib_index, u32 ti)
3428 {
3429   snat_main_t *sm = &snat_main;
3430   nat_ed_ses_key_t key;
3431   clib_bihash_kv_16_8_t kv, value;
3432   u32 thread_index;
3433   snat_session_t *s;
3434   snat_main_per_thread_data_t *tsm;
3435
3436   if (sm->num_workers > 1)
3437     thread_index =
3438       sm->first_worker_index +
3439       (sm->workers[(clib_net_to_host_u16 (out_port) -
3440                     1024) / sm->port_per_thread]);
3441   else
3442     thread_index = sm->num_workers;
3443   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3444
3445   key.l_addr.as_u32 = out_addr->as_u32;
3446   key.l_port = out_port;
3447   key.r_addr.as_u32 = eh_addr->as_u32;
3448   key.r_port = eh_port;
3449   key.proto = proto;
3450   key.fib_index = fib_index;
3451   kv.key[0] = key.as_u64[0];
3452   kv.key[1] = key.as_u64[1];
3453   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3454     return;
3455
3456   s = pool_elt_at_index (tsm->sessions, value.value);
3457   nat_free_session_data (sm, s, thread_index, 1);
3458   nat44_delete_session (sm, s, thread_index);
3459 }
3460
3461 void
3462 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3463                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3464                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3465                    u32 thread_index)
3466 {
3467   snat_main_t *sm = &snat_main;
3468   nat_ed_ses_key_t key;
3469   clib_bihash_kv_16_8_t kv, value;
3470   snat_session_t *s;
3471   snat_main_per_thread_data_t *tsm;
3472
3473   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3474
3475   key.l_addr.as_u32 = out_addr->as_u32;
3476   key.l_port = out_port;
3477   key.r_addr.as_u32 = eh_addr->as_u32;
3478   key.r_port = eh_port;
3479   key.proto = proto;
3480   key.fib_index = fib_index;
3481   kv.key[0] = key.as_u64[0];
3482   kv.key[1] = key.as_u64[1];
3483   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3484     return;
3485
3486   s = pool_elt_at_index (tsm->sessions, value.value);
3487   s->total_pkts = total_pkts;
3488   s->total_bytes = total_bytes;
3489 }
3490
3491 static clib_error_t *
3492 snat_config (vlib_main_t * vm, unformat_input_t * input)
3493 {
3494   snat_main_t *sm = &snat_main;
3495   nat66_main_t *nm = &nat66_main;
3496   u32 translation_buckets = 1024;
3497   u32 translation_memory_size = 128 << 20;
3498   u32 user_buckets = 128;
3499   u32 user_memory_size = 64 << 20;
3500   u32 max_translations_per_user = 100;
3501   u32 outside_vrf_id = 0;
3502   u32 outside_ip6_vrf_id = 0;
3503   u32 inside_vrf_id = 0;
3504   u32 static_mapping_buckets = 1024;
3505   u32 static_mapping_memory_size = 64 << 20;
3506   u32 nat64_bib_buckets = 1024;
3507   u32 nat64_bib_memory_size = 128 << 20;
3508   u32 nat64_st_buckets = 2048;
3509   u32 nat64_st_memory_size = 256 << 20;
3510   u8 static_mapping_only = 0;
3511   u8 static_mapping_connection_tracking = 0;
3512   snat_main_per_thread_data_t *tsm;
3513   dslite_main_t *dm = &dslite_main;
3514
3515   sm->deterministic = 0;
3516   sm->out2in_dpo = 0;
3517   sm->endpoint_dependent = 0;
3518
3519   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3520     {
3521       if (unformat
3522           (input, "translation hash buckets %d", &translation_buckets))
3523         ;
3524       else if (unformat (input, "translation hash memory %d",
3525                          &translation_memory_size));
3526       else if (unformat (input, "user hash buckets %d", &user_buckets))
3527         ;
3528       else if (unformat (input, "user hash memory %d", &user_memory_size))
3529         ;
3530       else if (unformat (input, "max translations per user %d",
3531                          &max_translations_per_user))
3532         ;
3533       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3534         ;
3535       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3536         ;
3537       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3538         ;
3539       else if (unformat (input, "static mapping only"))
3540         {
3541           static_mapping_only = 1;
3542           if (unformat (input, "connection tracking"))
3543             static_mapping_connection_tracking = 1;
3544         }
3545       else if (unformat (input, "deterministic"))
3546         sm->deterministic = 1;
3547       else if (unformat (input, "nat64 bib hash buckets %d",
3548                          &nat64_bib_buckets))
3549         ;
3550       else if (unformat (input, "nat64 bib hash memory %d",
3551                          &nat64_bib_memory_size))
3552         ;
3553       else
3554         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3555         ;
3556       else if (unformat (input, "nat64 st hash memory %d",
3557                          &nat64_st_memory_size))
3558         ;
3559       else if (unformat (input, "out2in dpo"))
3560         sm->out2in_dpo = 1;
3561       else if (unformat (input, "dslite ce"))
3562         dslite_set_ce (dm, 1);
3563       else if (unformat (input, "endpoint-dependent"))
3564         sm->endpoint_dependent = 1;
3565       else
3566         return clib_error_return (0, "unknown input '%U'",
3567                                   format_unformat_error, input);
3568     }
3569
3570   if (sm->deterministic && sm->endpoint_dependent)
3571     return clib_error_return (0,
3572                               "deterministic and endpoint-dependent modes are mutually exclusive");
3573
3574   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3575     return clib_error_return (0,
3576                               "static mapping only mode available only for simple nat");
3577
3578   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3579     return clib_error_return (0,
3580                               "out2in dpo mode available only for simple nat");
3581
3582   /* for show commands, etc. */
3583   sm->translation_buckets = translation_buckets;
3584   sm->translation_memory_size = translation_memory_size;
3585   /* do not exceed load factor 10 */
3586   sm->max_translations = 10 * translation_buckets;
3587   sm->user_buckets = user_buckets;
3588   sm->user_memory_size = user_memory_size;
3589   sm->max_translations_per_user = max_translations_per_user;
3590   sm->outside_vrf_id = outside_vrf_id;
3591   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3592                                                              outside_vrf_id,
3593                                                              FIB_SOURCE_PLUGIN_HI);
3594   nm->outside_vrf_id = outside_ip6_vrf_id;
3595   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3596                                                              outside_ip6_vrf_id,
3597                                                              FIB_SOURCE_PLUGIN_HI);
3598   sm->inside_vrf_id = inside_vrf_id;
3599   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3600                                                             inside_vrf_id,
3601                                                             FIB_SOURCE_PLUGIN_HI);
3602   sm->static_mapping_only = static_mapping_only;
3603   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3604
3605   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3606                   nat64_st_memory_size);
3607
3608   if (sm->deterministic)
3609     {
3610       sm->in2out_node_index = snat_det_in2out_node.index;
3611       sm->in2out_output_node_index = ~0;
3612       sm->out2in_node_index = snat_det_out2in_node.index;
3613       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3614       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3615     }
3616   else
3617     {
3618       if (sm->endpoint_dependent)
3619         {
3620           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3621           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3622           sm->in2out_node_index = nat44_ed_in2out_node.index;
3623           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3624           sm->out2in_node_index = nat44_ed_out2in_node.index;
3625           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3626           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3627           nat_affinity_init (vm);
3628           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
3629                        nat_ha_sref_ed_cb);
3630         }
3631       else
3632         {
3633           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3634           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3635           sm->in2out_node_index = snat_in2out_node.index;
3636           sm->in2out_output_node_index = snat_in2out_output_node.index;
3637           sm->out2in_node_index = snat_out2in_node.index;
3638           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3639           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3640           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
3641         }
3642       if (!static_mapping_only ||
3643           (static_mapping_only && static_mapping_connection_tracking))
3644         {
3645           /* *INDENT-OFF* */
3646           vec_foreach (tsm, sm->per_thread_data)
3647             {
3648               if (sm->endpoint_dependent)
3649                 {
3650                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3651                                          translation_buckets,
3652                                          translation_memory_size);
3653                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3654                                                       format_ed_session_kvp);
3655
3656                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3657                                          translation_buckets,
3658                                          translation_memory_size);
3659                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3660                                                       format_ed_session_kvp);
3661                 }
3662               else
3663                 {
3664                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3665                                         translation_buckets,
3666                                         translation_memory_size);
3667                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3668                                                      format_session_kvp);
3669
3670                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3671                                         translation_buckets,
3672                                         translation_memory_size);
3673                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3674                                                      format_session_kvp);
3675                 }
3676
3677               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3678                                     user_memory_size);
3679               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3680                                                  format_user_kvp);
3681             }
3682           /* *INDENT-ON* */
3683
3684         }
3685       else
3686         {
3687           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3688           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3689         }
3690       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3691                             "static_mapping_by_local", static_mapping_buckets,
3692                             static_mapping_memory_size);
3693       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3694                                          format_static_mapping_kvp);
3695
3696       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3697                             "static_mapping_by_external",
3698                             static_mapping_buckets,
3699                             static_mapping_memory_size);
3700       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3701                                          format_static_mapping_kvp);
3702     }
3703
3704   return 0;
3705 }
3706
3707 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3708
3709 static void
3710 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3711                                  uword opaque,
3712                                  u32 sw_if_index,
3713                                  ip4_address_t * address,
3714                                  u32 address_length,
3715                                  u32 if_address_index, u32 is_delete)
3716 {
3717   snat_main_t *sm = &snat_main;
3718   snat_static_map_resolve_t *rp;
3719   snat_static_mapping_t *m;
3720   snat_session_key_t m_key;
3721   clib_bihash_kv_8_8_t kv, value;
3722   int i, rv;
3723   ip4_address_t l_addr;
3724
3725   for (i = 0; i < vec_len (sm->to_resolve); i++)
3726     {
3727       rp = sm->to_resolve + i;
3728       if (rp->addr_only == 0)
3729         continue;
3730       if (rp->sw_if_index == sw_if_index)
3731         goto match;
3732     }
3733
3734   return;
3735
3736 match:
3737   m_key.addr.as_u32 = address->as_u32;
3738   m_key.port = rp->addr_only ? 0 : rp->e_port;
3739   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3740   m_key.fib_index = sm->outside_fib_index;
3741   kv.key = m_key.as_u64;
3742   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3743     m = 0;
3744   else
3745     m = pool_elt_at_index (sm->static_mappings, value.value);
3746
3747   if (!is_delete)
3748     {
3749       /* Don't trip over lease renewal, static config */
3750       if (m)
3751         return;
3752     }
3753   else
3754     {
3755       if (!m)
3756         return;
3757     }
3758
3759   /* Indetity mapping? */
3760   if (rp->l_addr.as_u32 == 0)
3761     l_addr.as_u32 = address[0].as_u32;
3762   else
3763     l_addr.as_u32 = rp->l_addr.as_u32;
3764   /* Add the static mapping */
3765   rv = snat_add_static_mapping (l_addr,
3766                                 address[0],
3767                                 rp->l_port,
3768                                 rp->e_port,
3769                                 rp->vrf_id,
3770                                 rp->addr_only, ~0 /* sw_if_index */ ,
3771                                 rp->proto, !is_delete, rp->twice_nat,
3772                                 rp->out2in_only, rp->tag, rp->identity_nat);
3773   if (rv)
3774     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3775 }
3776
3777 static void
3778 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3779                                        uword opaque,
3780                                        u32 sw_if_index,
3781                                        ip4_address_t * address,
3782                                        u32 address_length,
3783                                        u32 if_address_index, u32 is_delete)
3784 {
3785   snat_main_t *sm = &snat_main;
3786   snat_static_map_resolve_t *rp;
3787   ip4_address_t l_addr;
3788   int i, j;
3789   int rv;
3790   u8 twice_nat = 0;
3791   snat_address_t *addresses = sm->addresses;
3792
3793   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3794     {
3795       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3796         goto match;
3797     }
3798
3799   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3800     {
3801       twice_nat = 1;
3802       addresses = sm->twice_nat_addresses;
3803       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3804         goto match;
3805     }
3806
3807   return;
3808
3809 match:
3810   if (!is_delete)
3811     {
3812       /* Don't trip over lease renewal, static config */
3813       for (j = 0; j < vec_len (addresses); j++)
3814         if (addresses[j].addr.as_u32 == address->as_u32)
3815           return;
3816
3817       (void) snat_add_address (sm, address, ~0, twice_nat);
3818       /* Scan static map resolution vector */
3819       for (j = 0; j < vec_len (sm->to_resolve); j++)
3820         {
3821           rp = sm->to_resolve + j;
3822           if (rp->addr_only)
3823             continue;
3824           /* On this interface? */
3825           if (rp->sw_if_index == sw_if_index)
3826             {
3827               /* Indetity mapping? */
3828               if (rp->l_addr.as_u32 == 0)
3829                 l_addr.as_u32 = address[0].as_u32;
3830               else
3831                 l_addr.as_u32 = rp->l_addr.as_u32;
3832               /* Add the static mapping */
3833               rv = snat_add_static_mapping (l_addr,
3834                                             address[0],
3835                                             rp->l_port,
3836                                             rp->e_port,
3837                                             rp->vrf_id,
3838                                             rp->addr_only,
3839                                             ~0 /* sw_if_index */ ,
3840                                             rp->proto,
3841                                             rp->is_add, rp->twice_nat,
3842                                             rp->out2in_only, rp->tag,
3843                                             rp->identity_nat);
3844               if (rv)
3845                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3846             }
3847         }
3848       return;
3849     }
3850   else
3851     {
3852       (void) snat_del_address (sm, address[0], 1, twice_nat);
3853       return;
3854     }
3855 }
3856
3857
3858 int
3859 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3860                             u8 twice_nat)
3861 {
3862   ip4_main_t *ip4_main = sm->ip4_main;
3863   ip4_address_t *first_int_addr;
3864   snat_static_map_resolve_t *rp;
3865   u32 *indices_to_delete = 0;
3866   int i, j;
3867   u32 *auto_add_sw_if_indices =
3868     twice_nat ? sm->
3869     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3870
3871   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3872     );
3873
3874   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3875     {
3876       if (auto_add_sw_if_indices[i] == sw_if_index)
3877         {
3878           if (is_del)
3879             {
3880               /* if have address remove it */
3881               if (first_int_addr)
3882                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3883               else
3884                 {
3885                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3886                     {
3887                       rp = sm->to_resolve + j;
3888                       if (rp->sw_if_index == sw_if_index)
3889                         vec_add1 (indices_to_delete, j);
3890                     }
3891                   if (vec_len (indices_to_delete))
3892                     {
3893                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3894                         vec_del1 (sm->to_resolve, j);
3895                       vec_free (indices_to_delete);
3896                     }
3897                 }
3898               if (twice_nat)
3899                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3900               else
3901                 vec_del1 (sm->auto_add_sw_if_indices, i);
3902             }
3903           else
3904             return VNET_API_ERROR_VALUE_EXIST;
3905
3906           return 0;
3907         }
3908     }
3909
3910   if (is_del)
3911     return VNET_API_ERROR_NO_SUCH_ENTRY;
3912
3913   /* add to the auto-address list */
3914   if (twice_nat)
3915     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3916   else
3917     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3918
3919   /* If the address is already bound - or static - add it now */
3920   if (first_int_addr)
3921     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3922
3923   return 0;
3924 }
3925
3926 int
3927 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3928                    snat_protocol_t proto, u32 vrf_id, int is_in)
3929 {
3930   snat_main_per_thread_data_t *tsm;
3931   clib_bihash_kv_8_8_t kv, value;
3932   ip4_header_t ip;
3933   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3934   snat_session_key_t key;
3935   snat_session_t *s;
3936   clib_bihash_8_8_t *t;
3937
3938   if (sm->endpoint_dependent)
3939     return VNET_API_ERROR_UNSUPPORTED;
3940
3941   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3942   if (sm->num_workers > 1)
3943     tsm =
3944       vec_elt_at_index (sm->per_thread_data,
3945                         sm->worker_in2out_cb (&ip, fib_index));
3946   else
3947     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3948
3949   key.addr.as_u32 = addr->as_u32;
3950   key.port = clib_host_to_net_u16 (port);
3951   key.protocol = proto;
3952   key.fib_index = fib_index;
3953   kv.key = key.as_u64;
3954   t = is_in ? &tsm->in2out : &tsm->out2in;
3955   if (!clib_bihash_search_8_8 (t, &kv, &value))
3956     {
3957       if (pool_is_free_index (tsm->sessions, value.value))
3958         return VNET_API_ERROR_UNSPECIFIED;
3959
3960       s = pool_elt_at_index (tsm->sessions, value.value);
3961       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3962       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3963       return 0;
3964     }
3965
3966   return VNET_API_ERROR_NO_SUCH_ENTRY;
3967 }
3968
3969 int
3970 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3971                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3972                       u32 vrf_id, int is_in)
3973 {
3974   ip4_header_t ip;
3975   clib_bihash_16_8_t *t;
3976   nat_ed_ses_key_t key;
3977   clib_bihash_kv_16_8_t kv, value;
3978   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3979   snat_session_t *s;
3980   snat_main_per_thread_data_t *tsm;
3981
3982   if (!sm->endpoint_dependent)
3983     return VNET_API_ERROR_FEATURE_DISABLED;
3984
3985   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3986   if (sm->num_workers > 1)
3987     tsm =
3988       vec_elt_at_index (sm->per_thread_data,
3989                         sm->worker_in2out_cb (&ip, fib_index));
3990   else
3991     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3992
3993   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3994   key.l_addr.as_u32 = addr->as_u32;
3995   key.r_addr.as_u32 = eh_addr->as_u32;
3996   key.l_port = clib_host_to_net_u16 (port);
3997   key.r_port = clib_host_to_net_u16 (eh_port);
3998   key.proto = proto;
3999   key.fib_index = fib_index;
4000   kv.key[0] = key.as_u64[0];
4001   kv.key[1] = key.as_u64[1];
4002   if (clib_bihash_search_16_8 (t, &kv, &value))
4003     return VNET_API_ERROR_NO_SUCH_ENTRY;
4004
4005   if (pool_is_free_index (tsm->sessions, value.value))
4006     return VNET_API_ERROR_UNSPECIFIED;
4007   s = pool_elt_at_index (tsm->sessions, value.value);
4008   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4009   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4010   return 0;
4011 }
4012
4013 void
4014 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4015 {
4016   snat_main_t *sm = &snat_main;
4017
4018   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4019   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4020   sm->psid = psid;
4021   sm->psid_offset = psid_offset;
4022   sm->psid_length = psid_length;
4023 }
4024
4025 void
4026 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4027 {
4028   snat_main_t *sm = &snat_main;
4029
4030   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4031   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4032   sm->start_port = start_port;
4033   sm->end_port = end_port;
4034 }
4035
4036 void
4037 nat_set_alloc_addr_and_port_default (void)
4038 {
4039   snat_main_t *sm = &snat_main;
4040
4041   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4042   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4043 }
4044
4045 /*
4046  * fd.io coding-style-patch-verification: ON
4047  *
4048  * Local Variables:
4049  * eval: (c-set-style "gnu")
4050  * End:
4051  */