src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <vnet/fib/fib_table.h>
  35 #include <vnet/fib/ip4_fib.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 /* *INDENT-OFF* */
  42
  43 /* Hook up input features */
  44 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  45   .arc_name = "ip4-unicast",
  46   .node_name = "nat44-in2out",
  47   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  48 };
  49 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  50   .arc_name = "ip4-unicast",
  51   .node_name = "nat44-out2in",
  52   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  53                                "ip4-dhcp-client-detect"),
  54 };
  55 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  56   .arc_name = "ip4-unicast",
  57   .node_name = "nat44-classify",
  58   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  59 };
  60 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  61   .arc_name = "ip4-unicast",
  62   .node_name = "nat44-det-in2out",
  63   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  64 };
  65 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  66   .arc_name = "ip4-unicast",
  67   .node_name = "nat44-det-out2in",
  68   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  69                                "ip4-dhcp-client-detect"),
  70 };
  71 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  72   .arc_name = "ip4-unicast",
  73   .node_name = "nat44-det-classify",
  74   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  75 };
  76 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  77   .arc_name = "ip4-unicast",
  78   .node_name = "nat44-ed-in2out",
  79   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-ed-out2in",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  85                                "ip4-dhcp-client-detect"),
  86 };
  87 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  88   .arc_name = "ip4-unicast",
  89   .node_name = "nat44-ed-classify",
  90   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  91 };
  92 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-in2out-worker-handoff",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  96 };
  97 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-out2in-worker-handoff",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 101                                "ip4-dhcp-client-detect"),
 102 };
 103 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-handoff-classify",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 107 };
 108 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-in2out-fast",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 112 };
 113 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-out2in-fast",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 117                                "ip4-dhcp-client-detect"),
 118 };
 119 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-hairpin-dst",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 123 };
 124 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-ed-hairpin-dst",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 128 };
 129
 130 /* Hook up output features */
 131 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 132   .arc_name = "ip4-output",
 133   .node_name = "nat44-in2out-output",
 134   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 137   .arc_name = "ip4-output",
 138   .node_name = "nat44-in2out-output-worker-handoff",
 139   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 140 };
 141 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 142   .arc_name = "ip4-output",
 143   .node_name = "nat44-hairpin-src",
 144   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 145 };
 146 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-ed-in2out-output",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 150 };
 151 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-ed-hairpin-src",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 155 };
 156
 157 /* Hook up ip4-local features */
 158 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 159 {
 160   .arc_name = "ip4-local",
 161   .node_name = "nat44-hairpinning",
 162   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 163 };
 164 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 165 {
 166   .arc_name = "ip4-local",
 167   .node_name = "nat44-ed-hairpinning",
 168   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 169 };
 170
 171
 172 VLIB_PLUGIN_REGISTER () = {
 173     .version = VPP_BUILD_VER,
 174     .description = "Network Address Translation (NAT)",
 175 };
 176 /* *INDENT-ON* */
 177
 178 void
 179 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 180                        u8 is_ha)
 181 {
 182   snat_session_key_t key;
 183   clib_bihash_kv_8_8_t kv;
 184   nat_ed_ses_key_t ed_key;
 185   clib_bihash_kv_16_8_t ed_kv;
 186   snat_main_per_thread_data_t *tsm =
 187     vec_elt_at_index (sm->per_thread_data, thread_index);
 188
 189   if (is_fwd_bypass_session (s))
 190     {
 191       if (snat_is_unk_proto_session (s))
 192         {
 193           ed_key.proto = s->in2out.port;
 194           ed_key.r_port = 0;
 195           ed_key.l_port = 0;
 196         }
 197       else
 198         {
 199           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 200           ed_key.l_port = s->in2out.port;
 201           ed_key.r_port = s->ext_host_port;
 202         }
 203       ed_key.l_addr = s->in2out.addr;
 204       ed_key.r_addr = s->ext_host_addr;
 205       ed_key.fib_index = 0;
 206       ed_kv.key[0] = ed_key.as_u64[0];
 207       ed_kv.key[1] = ed_key.as_u64[1];
 208       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 209         nat_log_warn ("in2out_ed key del failed");
 210       return;
 211     }
 212
 213   /* session lookup tables */
 214   if (is_ed_session (s))
 215     {
 216       if (is_affinity_sessions (s))
 217         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 218                              s->in2out.protocol, s->out2in.port);
 219       ed_key.l_addr = s->out2in.addr;
 220       ed_key.r_addr = s->ext_host_addr;
 221       ed_key.fib_index = s->out2in.fib_index;
 222       if (snat_is_unk_proto_session (s))
 223         {
 224           ed_key.proto = s->in2out.port;
 225           ed_key.r_port = 0;
 226           ed_key.l_port = 0;
 227         }
 228       else
 229         {
 230           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 231           ed_key.l_port = s->out2in.port;
 232           ed_key.r_port = s->ext_host_port;
 233         }
 234       ed_kv.key[0] = ed_key.as_u64[0];
 235       ed_kv.key[1] = ed_key.as_u64[1];
 236       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 237         nat_log_warn ("out2in_ed key del failed");
 238       ed_key.l_addr = s->in2out.addr;
 239       ed_key.fib_index = s->in2out.fib_index;
 240       if (!snat_is_unk_proto_session (s))
 241         ed_key.l_port = s->in2out.port;
 242       if (is_twice_nat_session (s))
 243         {
 244           ed_key.r_addr = s->ext_host_nat_addr;
 245           ed_key.r_port = s->ext_host_nat_port;
 246         }
 247       ed_kv.key[0] = ed_key.as_u64[0];
 248       ed_kv.key[1] = ed_key.as_u64[1];
 249       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 250         nat_log_warn ("in2out_ed key del failed");
 251
 252       if (!is_ha)
 253         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 254                                &s->in2out.addr, s->in2out.port,
 255                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 256                                &s->out2in.addr, s->out2in.port,
 257                                &s->ext_host_addr, s->ext_host_port,
 258                                s->in2out.protocol, is_twice_nat_session (s));
 259     }
 260   else
 261     {
 262       kv.key = s->in2out.as_u64;
 263       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 264         nat_log_warn ("in2out key del failed");
 265       kv.key = s->out2in.as_u64;
 266       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 267         nat_log_warn ("out2in key del failed");
 268
 269       if (!is_ha)
 270         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 271                                  &s->in2out.addr, s->in2out.port,
 272                                  &s->out2in.addr, s->out2in.port,
 273                                  s->in2out.protocol);
 274     }
 275
 276   if (snat_is_unk_proto_session (s))
 277     return;
 278
 279   if (!is_ha)
 280     {
 281       /* log NAT event */
 282       snat_ipfix_logging_nat44_ses_delete (thread_index,
 283                                            s->in2out.addr.as_u32,
 284                                            s->out2in.addr.as_u32,
 285                                            s->in2out.protocol,
 286                                            s->in2out.port,
 287                                            s->out2in.port,
 288                                            s->in2out.fib_index);
 289
 290       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 291                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 292                    thread_index);
 293     }
 294
 295   /* Twice NAT address and port for external host */
 296   if (is_twice_nat_session (s))
 297     {
 298       key.protocol = s->in2out.protocol;
 299       key.port = s->ext_host_nat_port;
 300       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 301       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 302                                           thread_index, &key);
 303     }
 304
 305   if (snat_is_session_static (s))
 306     return;
 307
 308   snat_free_outside_address_and_port (sm->addresses, thread_index,
 309                                       &s->out2in);
 310 }
 311
 312 snat_user_t *
 313 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 314                         u32 thread_index)
 315 {
 316   snat_user_t *u = 0;
 317   snat_user_key_t user_key;
 318   clib_bihash_kv_8_8_t kv, value;
 319   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 320   dlist_elt_t *per_user_list_head_elt;
 321
 322   user_key.addr.as_u32 = addr->as_u32;
 323   user_key.fib_index = fib_index;
 324   kv.key = user_key.as_u64;
 325
 326   /* Ever heard of the "user" = src ip4 address before? */
 327   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 328     {
 329       /* no, make a new one */
 330       pool_get (tsm->users, u);
 331       clib_memset (u, 0, sizeof (*u));
 332       u->addr.as_u32 = addr->as_u32;
 333       u->fib_index = fib_index;
 334
 335       pool_get (tsm->list_pool, per_user_list_head_elt);
 336
 337       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 338         tsm->list_pool;
 339
 340       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 341
 342       kv.value = u - tsm->users;
 343
 344       /* add user */
 345       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 346         nat_log_warn ("user_hash keay add failed");
 347
 348       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 349                                pool_elts (tsm->users));
 350     }
 351   else
 352     {
 353       u = pool_elt_at_index (tsm->users, value.value);
 354     }
 355
 356   return u;
 357 }
 358
 359 snat_session_t *
 360 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 361                               u32 thread_index, f64 now)
 362 {
 363   snat_session_t *s;
 364   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 365   u32 oldest_per_user_translation_list_index, session_index;
 366   dlist_elt_t *oldest_per_user_translation_list_elt;
 367   dlist_elt_t *per_user_translation_list_elt;
 368
 369   /* Over quota? Recycle the least recently used translation */
 370   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 371     {
 372       oldest_per_user_translation_list_index =
 373         clib_dlist_remove_head (tsm->list_pool,
 374                                 u->sessions_per_user_list_head_index);
 375
 376       ASSERT (oldest_per_user_translation_list_index != ~0);
 377
 378       /* Add it back to the end of the LRU list */
 379       clib_dlist_addtail (tsm->list_pool,
 380                           u->sessions_per_user_list_head_index,
 381                           oldest_per_user_translation_list_index);
 382       /* Get the list element */
 383       oldest_per_user_translation_list_elt =
 384         pool_elt_at_index (tsm->list_pool,
 385                            oldest_per_user_translation_list_index);
 386
 387       /* Get the session index from the list element */
 388       session_index = oldest_per_user_translation_list_elt->value;
 389
 390       /* Get the session */
 391       s = pool_elt_at_index (tsm->sessions, session_index);
 392       nat_free_session_data (sm, s, thread_index, 0);
 393       if (snat_is_session_static (s))
 394         u->nstaticsessions--;
 395       else
 396         u->nsessions--;
 397       s->flags = 0;
 398       s->total_bytes = 0;
 399       s->total_pkts = 0;
 400       s->state = 0;
 401       s->ext_host_addr.as_u32 = 0;
 402       s->ext_host_port = 0;
 403       s->ext_host_nat_addr.as_u32 = 0;
 404       s->ext_host_nat_port = 0;
 405     }
 406   else
 407     {
 408       pool_get (tsm->sessions, s);
 409       clib_memset (s, 0, sizeof (*s));
 410
 411       /* Create list elts */
 412       pool_get (tsm->list_pool, per_user_translation_list_elt);
 413       clib_dlist_init (tsm->list_pool,
 414                        per_user_translation_list_elt - tsm->list_pool);
 415
 416       per_user_translation_list_elt->value = s - tsm->sessions;
 417       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 418       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 419
 420       clib_dlist_addtail (tsm->list_pool,
 421                           s->per_user_list_head_index,
 422                           per_user_translation_list_elt - tsm->list_pool);
 423
 424       s->user_index = u - tsm->users;
 425       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 426                                pool_elts (tsm->sessions));
 427     }
 428
 429   s->ha_last_refreshed = now;
 430
 431   return s;
 432 }
 433
 434 snat_session_t *
 435 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 436                       f64 now)
 437 {
 438   snat_session_t *s;
 439   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 440   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 441   u32 oldest_index;
 442   u64 sess_timeout_time;
 443
 444   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 445     goto alloc_new;
 446
 447   oldest_index =
 448     clib_dlist_remove_head (tsm->list_pool,
 449                             u->sessions_per_user_list_head_index);
 450   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 451   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 452   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 453   if (now >= sess_timeout_time)
 454     {
 455       clib_dlist_addtail (tsm->list_pool,
 456                           u->sessions_per_user_list_head_index, oldest_index);
 457       nat_free_session_data (sm, s, thread_index, 0);
 458       if (snat_is_session_static (s))
 459         u->nstaticsessions--;
 460       else
 461         u->nsessions--;
 462       s->flags = 0;
 463       s->total_bytes = 0;
 464       s->total_pkts = 0;
 465       s->state = 0;
 466       s->ext_host_addr.as_u32 = 0;
 467       s->ext_host_port = 0;
 468       s->ext_host_nat_addr.as_u32 = 0;
 469       s->ext_host_nat_port = 0;
 470     }
 471   else
 472     {
 473       clib_dlist_addhead (tsm->list_pool,
 474                           u->sessions_per_user_list_head_index, oldest_index);
 475       if ((u->nsessions + u->nstaticsessions) >=
 476           sm->max_translations_per_user)
 477         {
 478           nat_log_warn ("max translations per user %U", format_ip4_address,
 479                         &u->addr);
 480           snat_ipfix_logging_max_entries_per_user
 481             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 482           return 0;
 483         }
 484       else
 485         {
 486         alloc_new:
 487           pool_get (tsm->sessions, s);
 488           clib_memset (s, 0, sizeof (*s));
 489
 490           /* Create list elts */
 491           pool_get (tsm->list_pool, per_user_translation_list_elt);
 492           clib_dlist_init (tsm->list_pool,
 493                            per_user_translation_list_elt - tsm->list_pool);
 494
 495           per_user_translation_list_elt->value = s - tsm->sessions;
 496           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 497           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 498
 499           clib_dlist_addtail (tsm->list_pool,
 500                               s->per_user_list_head_index,
 501                               per_user_translation_list_elt - tsm->list_pool);
 502         }
 503
 504       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 505                                pool_elts (tsm->sessions));
 506     }
 507
 508   s->ha_last_refreshed = now;
 509
 510   return s;
 511 }
 512
 513 void
 514 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 515                           int is_add)
 516 {
 517   fib_prefix_t prefix = {
 518     .fp_len = p_len,
 519     .fp_proto = FIB_PROTOCOL_IP4,
 520     .fp_addr = {
 521                 .ip4.as_u32 = addr->as_u32,
 522                 },
 523   };
 524   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 525
 526   if (is_add)
 527     fib_table_entry_update_one_path (fib_index,
 528                                      &prefix,
 529                                      FIB_SOURCE_PLUGIN_LOW,
 530                                      (FIB_ENTRY_FLAG_CONNECTED |
 531                                       FIB_ENTRY_FLAG_LOCAL |
 532                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 533                                      DPO_PROTO_IP4,
 534                                      NULL,
 535                                      sw_if_index,
 536                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 537   else
 538     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 539 }
 540
 541 int
 542 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 543                   u8 twice_nat)
 544 {
 545   snat_address_t *ap;
 546   snat_interface_t *i;
 547   vlib_thread_main_t *tm = vlib_get_thread_main ();
 548
 549   if (twice_nat && !sm->endpoint_dependent)
 550     return VNET_API_ERROR_FEATURE_DISABLED;
 551
 552   /* Check if address already exists */
 553   /* *INDENT-OFF* */
 554   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 555     {
 556       if (ap->addr.as_u32 == addr->as_u32)
 557         return VNET_API_ERROR_VALUE_EXIST;
 558     }
 559   /* *INDENT-ON* */
 560
 561   if (twice_nat)
 562     vec_add2 (sm->twice_nat_addresses, ap, 1);
 563   else
 564     vec_add2 (sm->addresses, ap, 1);
 565
 566   ap->addr = *addr;
 567   if (vrf_id != ~0)
 568     ap->fib_index =
 569       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 570                                          FIB_SOURCE_PLUGIN_LOW);
 571   else
 572     ap->fib_index = ~0;
 573 #define _(N, i, n, s) \
 574   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 575   ap->busy_##n##_ports = 0; \
 576   ap->busy_##n##_ports_per_thread = 0;\
 577   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 578   foreach_snat_protocol
 579 #undef _
 580     if (twice_nat)
 581     return 0;
 582
 583   /* Add external address to FIB */
 584   /* *INDENT-OFF* */
 585   pool_foreach (i, sm->interfaces,
 586   ({
 587     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 588       continue;
 589
 590     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 591     break;
 592   }));
 593   pool_foreach (i, sm->output_feature_interfaces,
 594   ({
 595     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 596       continue;
 597
 598     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 599     break;
 600   }));
 601   /* *INDENT-ON* */
 602
 603   return 0;
 604 }
 605
 606 static int
 607 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 608 {
 609   snat_static_mapping_t *m;
 610   /* *INDENT-OFF* */
 611   pool_foreach (m, sm->static_mappings,
 612   ({
 613       if (is_addr_only_static_mapping (m) ||
 614           is_out2in_only_static_mapping (m) ||
 615           is_identity_static_mapping (m))
 616         continue;
 617       if (m->external_addr.as_u32 == addr.as_u32)
 618         return 1;
 619   }));
 620   /* *INDENT-ON* */
 621
 622   return 0;
 623 }
 624
 625 void
 626 increment_v4_address (ip4_address_t * a)
 627 {
 628   u32 v;
 629
 630   v = clib_net_to_host_u32 (a->as_u32) + 1;
 631   a->as_u32 = clib_host_to_net_u32 (v);
 632 }
 633
 634 static void
 635 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 636                                        ip4_address_t l_addr,
 637                                        u16 l_port,
 638                                        u32 sw_if_index,
 639                                        u16 e_port,
 640                                        u32 vrf_id,
 641                                        snat_protocol_t proto,
 642                                        int addr_only, int is_add, u8 * tag,
 643                                        int twice_nat, int out2in_only,
 644                                        int identity_nat)
 645 {
 646   snat_static_map_resolve_t *rp;
 647
 648   vec_add2 (sm->to_resolve, rp, 1);
 649   rp->l_addr.as_u32 = l_addr.as_u32;
 650   rp->l_port = l_port;
 651   rp->sw_if_index = sw_if_index;
 652   rp->e_port = e_port;
 653   rp->vrf_id = vrf_id;
 654   rp->proto = proto;
 655   rp->addr_only = addr_only;
 656   rp->is_add = is_add;
 657   rp->twice_nat = twice_nat;
 658   rp->out2in_only = out2in_only;
 659   rp->identity_nat = identity_nat;
 660   rp->tag = vec_dup (tag);
 661 }
 662
 663 static u32
 664 get_thread_idx_by_port (u16 e_port)
 665 {
 666   snat_main_t *sm = &snat_main;
 667   u32 thread_idx = sm->num_workers;
 668   if (sm->num_workers > 1)
 669     {
 670       thread_idx =
 671         sm->first_worker_index +
 672         sm->workers[(e_port - 1024) / sm->port_per_thread];
 673     }
 674   return thread_idx;
 675 }
 676
 677 int
 678 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 679                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 680                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 681                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 682                          u8 identity_nat)
 683 {
 684   snat_main_t *sm = &snat_main;
 685   snat_static_mapping_t *m;
 686   snat_session_key_t m_key;
 687   clib_bihash_kv_8_8_t kv, value;
 688   snat_address_t *a = 0;
 689   u32 fib_index = ~0;
 690   snat_interface_t *interface;
 691   int i;
 692   snat_main_per_thread_data_t *tsm;
 693   snat_user_key_t u_key;
 694   snat_user_t *u;
 695   dlist_elt_t *head, *elt;
 696   u32 elt_index, head_index;
 697   u32 ses_index;
 698   u64 user_index;
 699   snat_session_t *s;
 700   snat_static_map_resolve_t *rp, *rp_match = 0;
 701   nat44_lb_addr_port_t *local;
 702   u32 find = ~0;
 703
 704   if (!sm->endpoint_dependent)
 705     {
 706       if (twice_nat || out2in_only)
 707         return VNET_API_ERROR_FEATURE_DISABLED;
 708     }
 709
 710   /* If the external address is a specific interface address */
 711   if (sw_if_index != ~0)
 712     {
 713       ip4_address_t *first_int_addr;
 714
 715       for (i = 0; i < vec_len (sm->to_resolve); i++)
 716         {
 717           rp = sm->to_resolve + i;
 718           if (rp->sw_if_index != sw_if_index ||
 719               rp->l_addr.as_u32 != l_addr.as_u32 ||
 720               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 721             continue;
 722
 723           if (!addr_only)
 724             {
 725               if ((rp->l_port != l_port && rp->e_port != e_port)
 726                   || rp->proto != proto)
 727                 continue;
 728             }
 729
 730           rp_match = rp;
 731           break;
 732         }
 733
 734       /* Might be already set... */
 735       first_int_addr = ip4_interface_first_address
 736         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 737
 738       if (is_add)
 739         {
 740           if (rp_match)
 741             return VNET_API_ERROR_VALUE_EXIST;
 742
 743           snat_add_static_mapping_when_resolved
 744             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 745              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 746
 747           /* DHCP resolution required? */
 748           if (first_int_addr == 0)
 749             {
 750               return 0;
 751             }
 752           else
 753             {
 754               e_addr.as_u32 = first_int_addr->as_u32;
 755               /* Identity mapping? */
 756               if (l_addr.as_u32 == 0)
 757                 l_addr.as_u32 = e_addr.as_u32;
 758             }
 759         }
 760       else
 761         {
 762           if (!rp_match)
 763             return VNET_API_ERROR_NO_SUCH_ENTRY;
 764
 765           vec_del1 (sm->to_resolve, i);
 766
 767           if (first_int_addr)
 768             {
 769               e_addr.as_u32 = first_int_addr->as_u32;
 770               /* Identity mapping? */
 771               if (l_addr.as_u32 == 0)
 772                 l_addr.as_u32 = e_addr.as_u32;
 773             }
 774           else
 775             return 0;
 776         }
 777     }
 778
 779   m_key.addr = e_addr;
 780   m_key.port = addr_only ? 0 : e_port;
 781   m_key.protocol = addr_only ? 0 : proto;
 782   m_key.fib_index = 0;
 783   kv.key = m_key.as_u64;
 784   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 785     m = 0;
 786   else
 787     m = pool_elt_at_index (sm->static_mappings, value.value);
 788
 789   if (is_add)
 790     {
 791       if (m)
 792         {
 793           if (is_identity_static_mapping (m))
 794             {
 795               /* *INDENT-OFF* */
 796               pool_foreach (local, m->locals,
 797               ({
 798                 if (local->vrf_id == vrf_id)
 799                   return VNET_API_ERROR_VALUE_EXIST;
 800               }));
 801               /* *INDENT-ON* */
 802               pool_get (m->locals, local);
 803               local->vrf_id = vrf_id;
 804               local->fib_index =
 805                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 806                                                    FIB_SOURCE_PLUGIN_LOW);
 807               m_key.addr = m->local_addr;
 808               m_key.port = m->local_port;
 809               m_key.protocol = m->proto;
 810               m_key.fib_index = local->fib_index;
 811               kv.key = m_key.as_u64;
 812               kv.value = m - sm->static_mappings;
 813               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 814               return 0;
 815             }
 816           else
 817             return VNET_API_ERROR_VALUE_EXIST;
 818         }
 819
 820       if (twice_nat && addr_only)
 821         return VNET_API_ERROR_UNSUPPORTED;
 822
 823       /* Convert VRF id to FIB index */
 824       if (vrf_id != ~0)
 825         fib_index =
 826           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 827                                              FIB_SOURCE_PLUGIN_LOW);
 828       /* If not specified use inside VRF id from SNAT plugin startup config */
 829       else
 830         {
 831           fib_index = sm->inside_fib_index;
 832           vrf_id = sm->inside_vrf_id;
 833         }
 834
 835       if (!(out2in_only || identity_nat))
 836         {
 837           m_key.addr = l_addr;
 838           m_key.port = addr_only ? 0 : l_port;
 839           m_key.protocol = addr_only ? 0 : proto;
 840           m_key.fib_index = fib_index;
 841           kv.key = m_key.as_u64;
 842           if (!clib_bihash_search_8_8
 843               (&sm->static_mapping_by_local, &kv, &value))
 844             return VNET_API_ERROR_VALUE_EXIST;
 845         }
 846
 847       /* Find external address in allocated addresses and reserve port for
 848          address and port pair mapping when dynamic translations enabled */
 849       if (!(addr_only || sm->static_mapping_only || out2in_only))
 850         {
 851           for (i = 0; i < vec_len (sm->addresses); i++)
 852             {
 853               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 854                 {
 855                   a = sm->addresses + i;
 856                   /* External port must be unused */
 857                   switch (proto)
 858                     {
 859 #define _(N, j, n, s) \
 860                     case SNAT_PROTOCOL_##N: \
 861                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 862                         return VNET_API_ERROR_INVALID_VALUE; \
 863                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 864                       if (e_port > 1024) \
 865                         { \
 866                           a->busy_##n##_ports++; \
 867                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 868                         } \
 869                       break;
 870                       foreach_snat_protocol
 871 #undef _
 872                     default:
 873                       nat_log_info ("unknown protocol");
 874                       return VNET_API_ERROR_INVALID_VALUE_2;
 875                     }
 876                   break;
 877                 }
 878             }
 879           /* External address must be allocated */
 880           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 881             {
 882               if (sw_if_index != ~0)
 883                 {
 884                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 885                     {
 886                       rp = sm->to_resolve + i;
 887                       if (rp->addr_only)
 888                         continue;
 889                       if (rp->sw_if_index != sw_if_index &&
 890                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 891                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 892                           rp->e_port != e_port && rp->proto != proto)
 893                         continue;
 894
 895                       vec_del1 (sm->to_resolve, i);
 896                       break;
 897                     }
 898                 }
 899               return VNET_API_ERROR_NO_SUCH_ENTRY;
 900             }
 901         }
 902
 903       pool_get (sm->static_mappings, m);
 904       clib_memset (m, 0, sizeof (*m));
 905       m->tag = vec_dup (tag);
 906       m->local_addr = l_addr;
 907       m->external_addr = e_addr;
 908       m->twice_nat = twice_nat;
 909       if (out2in_only)
 910         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 911       if (addr_only)
 912         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 913       if (identity_nat)
 914         {
 915           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 916           pool_get (m->locals, local);
 917           local->vrf_id = vrf_id;
 918           local->fib_index = fib_index;
 919         }
 920       else
 921         {
 922           m->vrf_id = vrf_id;
 923           m->fib_index = fib_index;
 924         }
 925       if (!addr_only)
 926         {
 927           m->local_port = l_port;
 928           m->external_port = e_port;
 929           m->proto = proto;
 930         }
 931
 932       if (sm->num_workers > 1)
 933         {
 934           ip4_header_t ip = {
 935             .src_address = m->local_addr,
 936           };
 937           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 938           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 939         }
 940       else
 941         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 942
 943       m_key.addr = m->local_addr;
 944       m_key.port = m->local_port;
 945       m_key.protocol = m->proto;
 946       m_key.fib_index = fib_index;
 947       kv.key = m_key.as_u64;
 948       kv.value = m - sm->static_mappings;
 949       if (!out2in_only)
 950         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 951
 952       m_key.addr = m->external_addr;
 953       m_key.port = m->external_port;
 954       m_key.fib_index = 0;
 955       kv.key = m_key.as_u64;
 956       kv.value = m - sm->static_mappings;
 957       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 958
 959       /* Delete dynamic sessions matching local address (+ local port) */
 960       if (!(sm->static_mapping_only))
 961         {
 962           u_key.addr = m->local_addr;
 963           u_key.fib_index = m->fib_index;
 964           kv.key = u_key.as_u64;
 965           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 966             {
 967               user_index = value.value;
 968               u = pool_elt_at_index (tsm->users, user_index);
 969               if (u->nsessions)
 970                 {
 971                   head_index = u->sessions_per_user_list_head_index;
 972                   head = pool_elt_at_index (tsm->list_pool, head_index);
 973                   elt_index = head->next;
 974                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 975                   ses_index = elt->value;
 976                   while (ses_index != ~0)
 977                     {
 978                       s = pool_elt_at_index (tsm->sessions, ses_index);
 979                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 980                       ses_index = elt->value;
 981
 982                       if (snat_is_session_static (s))
 983                         continue;
 984
 985                       if (!addr_only
 986                           && (clib_net_to_host_u16 (s->in2out.port) !=
 987                               m->local_port))
 988                         continue;
 989
 990                       nat_free_session_data (sm, s,
 991                                              tsm - sm->per_thread_data, 0);
 992                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 993
 994                       if (!addr_only && !sm->endpoint_dependent)
 995                         break;
 996                     }
 997                 }
 998             }
 999         }
1000     }
1001   else
1002     {
1003       if (!m)
1004         {
1005           if (sw_if_index != ~0)
1006             return 0;
1007           else
1008             return VNET_API_ERROR_NO_SUCH_ENTRY;
1009         }
1010
1011       if (identity_nat)
1012         {
1013           if (vrf_id == ~0)
1014             vrf_id = sm->inside_vrf_id;
1015
1016           /* *INDENT-OFF* */
1017           pool_foreach (local, m->locals,
1018           ({
1019             if (local->vrf_id == vrf_id)
1020               find = local - m->locals;
1021           }));
1022           /* *INDENT-ON* */
1023           if (find == ~0)
1024             return VNET_API_ERROR_NO_SUCH_ENTRY;
1025
1026           local = pool_elt_at_index (m->locals, find);
1027           fib_index = local->fib_index;
1028           pool_put (m->locals, local);
1029         }
1030       else
1031         fib_index = m->fib_index;
1032
1033       /* Free external address port */
1034       if (!(addr_only || sm->static_mapping_only || out2in_only))
1035         {
1036           for (i = 0; i < vec_len (sm->addresses); i++)
1037             {
1038               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1039                 {
1040                   a = sm->addresses + i;
1041                   switch (proto)
1042                     {
1043 #define _(N, j, n, s) \
1044                     case SNAT_PROTOCOL_##N: \
1045                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1046                       if (e_port > 1024) \
1047                         { \
1048                           a->busy_##n##_ports--; \
1049                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1050                         } \
1051                       break;
1052                       foreach_snat_protocol
1053 #undef _
1054                     default:
1055                       nat_log_info ("unknown protocol");
1056                       return VNET_API_ERROR_INVALID_VALUE_2;
1057                     }
1058                   break;
1059                 }
1060             }
1061         }
1062
1063       if (sm->num_workers > 1)
1064         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1065       else
1066         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1067
1068       m_key.addr = m->local_addr;
1069       m_key.port = m->local_port;
1070       m_key.protocol = m->proto;
1071       m_key.fib_index = fib_index;
1072       kv.key = m_key.as_u64;
1073       if (!out2in_only)
1074         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1075
1076       /* Delete session(s) for static mapping if exist */
1077       if (!(sm->static_mapping_only) ||
1078           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1079         {
1080           u_key.addr = m->local_addr;
1081           u_key.fib_index = fib_index;
1082           kv.key = u_key.as_u64;
1083           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1084             {
1085               user_index = value.value;
1086               u = pool_elt_at_index (tsm->users, user_index);
1087               if (u->nstaticsessions)
1088                 {
1089                   head_index = u->sessions_per_user_list_head_index;
1090                   head = pool_elt_at_index (tsm->list_pool, head_index);
1091                   elt_index = head->next;
1092                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1093                   ses_index = elt->value;
1094                   while (ses_index != ~0)
1095                     {
1096                       s = pool_elt_at_index (tsm->sessions, ses_index);
1097                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1098                       ses_index = elt->value;
1099
1100                       if (!addr_only)
1101                         {
1102                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1103                               (clib_net_to_host_u16 (s->out2in.port) !=
1104                                e_port))
1105                             continue;
1106                         }
1107
1108                       if (is_lb_session (s))
1109                         continue;
1110
1111                       if (!snat_is_session_static (s))
1112                         continue;
1113
1114                       nat_free_session_data (sm, s,
1115                                              tsm - sm->per_thread_data, 0);
1116                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1117
1118                       if (!addr_only && !sm->endpoint_dependent)
1119                         break;
1120                     }
1121                 }
1122             }
1123         }
1124
1125       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1126       if (pool_elts (m->locals))
1127         return 0;
1128
1129       m_key.addr = m->external_addr;
1130       m_key.port = m->external_port;
1131       m_key.fib_index = 0;
1132       kv.key = m_key.as_u64;
1133       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1134
1135       vec_free (m->tag);
1136       vec_free (m->workers);
1137       /* Delete static mapping from pool */
1138       pool_put (sm->static_mappings, m);
1139     }
1140
1141   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1142     return 0;
1143
1144   /* Add/delete external address to FIB */
1145   /* *INDENT-OFF* */
1146   pool_foreach (interface, sm->interfaces,
1147   ({
1148     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1149       continue;
1150
1151     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1152     break;
1153   }));
1154   pool_foreach (interface, sm->output_feature_interfaces,
1155   ({
1156     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1157       continue;
1158
1159     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1160     break;
1161   }));
1162   /* *INDENT-ON* */
1163
1164   return 0;
1165 }
1166
1167 int
1168 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1169                                  snat_protocol_t proto,
1170                                  nat44_lb_addr_port_t * locals, u8 is_add,
1171                                  twice_nat_type_t twice_nat, u8 out2in_only,
1172                                  u8 * tag, u32 affinity)
1173 {
1174   snat_main_t *sm = &snat_main;
1175   snat_static_mapping_t *m;
1176   snat_session_key_t m_key;
1177   clib_bihash_kv_8_8_t kv, value;
1178   snat_address_t *a = 0;
1179   int i;
1180   nat44_lb_addr_port_t *local;
1181   u32 elt_index, head_index, ses_index;
1182   snat_main_per_thread_data_t *tsm;
1183   snat_user_key_t u_key;
1184   snat_user_t *u;
1185   snat_session_t *s;
1186   dlist_elt_t *head, *elt;
1187   uword *bitmap = 0;
1188
1189   if (!sm->endpoint_dependent)
1190     return VNET_API_ERROR_FEATURE_DISABLED;
1191
1192   m_key.addr = e_addr;
1193   m_key.port = e_port;
1194   m_key.protocol = proto;
1195   m_key.fib_index = 0;
1196   kv.key = m_key.as_u64;
1197   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1198     m = 0;
1199   else
1200     m = pool_elt_at_index (sm->static_mappings, value.value);
1201
1202   if (is_add)
1203     {
1204       if (m)
1205         return VNET_API_ERROR_VALUE_EXIST;
1206
1207       if (vec_len (locals) < 2)
1208         return VNET_API_ERROR_INVALID_VALUE;
1209
1210       /* Find external address in allocated addresses and reserve port for
1211          address and port pair mapping when dynamic translations enabled */
1212       if (!(sm->static_mapping_only || out2in_only))
1213         {
1214           for (i = 0; i < vec_len (sm->addresses); i++)
1215             {
1216               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1217                 {
1218                   a = sm->addresses + i;
1219                   /* External port must be unused */
1220                   switch (proto)
1221                     {
1222 #define _(N, j, n, s) \
1223                     case SNAT_PROTOCOL_##N: \
1224                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1225                         return VNET_API_ERROR_INVALID_VALUE; \
1226                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1227                       if (e_port > 1024) \
1228                         { \
1229                           a->busy_##n##_ports++; \
1230                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1231                         } \
1232                       break;
1233                       foreach_snat_protocol
1234 #undef _
1235                     default:
1236                       nat_log_info ("unknown protocol");
1237                       return VNET_API_ERROR_INVALID_VALUE_2;
1238                     }
1239                   break;
1240                 }
1241             }
1242           /* External address must be allocated */
1243           if (!a)
1244             return VNET_API_ERROR_NO_SUCH_ENTRY;
1245         }
1246
1247       pool_get (sm->static_mappings, m);
1248       clib_memset (m, 0, sizeof (*m));
1249       m->tag = vec_dup (tag);
1250       m->external_addr = e_addr;
1251       m->external_port = e_port;
1252       m->proto = proto;
1253       m->twice_nat = twice_nat;
1254       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1255       if (out2in_only)
1256         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1257       m->affinity = affinity;
1258
1259       if (affinity)
1260         m->affinity_per_service_list_head_index =
1261           nat_affinity_get_per_service_list_head_index ();
1262       else
1263         m->affinity_per_service_list_head_index = ~0;
1264
1265       m_key.addr = m->external_addr;
1266       m_key.port = m->external_port;
1267       m_key.protocol = m->proto;
1268       m_key.fib_index = 0;
1269       kv.key = m_key.as_u64;
1270       kv.value = m - sm->static_mappings;
1271       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1272         {
1273           nat_log_err ("static_mapping_by_external key add failed");
1274           return VNET_API_ERROR_UNSPECIFIED;
1275         }
1276
1277       m_key.fib_index = m->fib_index;
1278       for (i = 0; i < vec_len (locals); i++)
1279         {
1280           locals[i].fib_index =
1281             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1282                                                locals[i].vrf_id,
1283                                                FIB_SOURCE_PLUGIN_LOW);
1284           m_key.addr = locals[i].addr;
1285           m_key.fib_index = locals[i].fib_index;
1286           if (!out2in_only)
1287             {
1288               m_key.port = locals[i].port;
1289               kv.key = m_key.as_u64;
1290               kv.value = m - sm->static_mappings;
1291               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1292             }
1293           locals[i].prefix = (i == 0) ? locals[i].probability :
1294             (locals[i - 1].prefix + locals[i].probability);
1295           pool_get (m->locals, local);
1296           *local = locals[i];
1297           if (sm->num_workers > 1)
1298             {
1299               ip4_header_t ip = {
1300                 .src_address = locals[i].addr,
1301               };
1302               bitmap =
1303                 clib_bitmap_set (bitmap,
1304                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1305             }
1306         }
1307
1308       /* Assign workers */
1309       if (sm->num_workers > 1)
1310         {
1311           /* *INDENT-OFF* */
1312           clib_bitmap_foreach (i, bitmap,
1313             ({
1314                vec_add1(m->workers, i);
1315             }));
1316           /* *INDENT-ON* */
1317         }
1318     }
1319   else
1320     {
1321       if (!m)
1322         return VNET_API_ERROR_NO_SUCH_ENTRY;
1323
1324       if (!is_lb_static_mapping (m))
1325         return VNET_API_ERROR_INVALID_VALUE;
1326
1327       /* Free external address port */
1328       if (!(sm->static_mapping_only || out2in_only))
1329         {
1330           for (i = 0; i < vec_len (sm->addresses); i++)
1331             {
1332               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1333                 {
1334                   a = sm->addresses + i;
1335                   switch (proto)
1336                     {
1337 #define _(N, j, n, s) \
1338                     case SNAT_PROTOCOL_##N: \
1339                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1340                       if (e_port > 1024) \
1341                         { \
1342                           a->busy_##n##_ports--; \
1343                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1344                         } \
1345                       break;
1346                       foreach_snat_protocol
1347 #undef _
1348                     default:
1349                       nat_log_info ("unknown protocol");
1350                       return VNET_API_ERROR_INVALID_VALUE_2;
1351                     }
1352                   break;
1353                 }
1354             }
1355         }
1356
1357       m_key.addr = m->external_addr;
1358       m_key.port = m->external_port;
1359       m_key.protocol = m->proto;
1360       m_key.fib_index = 0;
1361       kv.key = m_key.as_u64;
1362       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1363         {
1364           nat_log_err ("static_mapping_by_external key del failed");
1365           return VNET_API_ERROR_UNSPECIFIED;
1366         }
1367
1368       /* *INDENT-OFF* */
1369       pool_foreach (local, m->locals,
1370       ({
1371           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1372                             FIB_SOURCE_PLUGIN_LOW);
1373           m_key.addr = local->addr;
1374           if (!out2in_only)
1375             {
1376               m_key.port = local->port;
1377               m_key.fib_index = local->fib_index;
1378               kv.key = m_key.as_u64;
1379               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1380                 {
1381                   nat_log_err ("static_mapping_by_local key del failed");
1382                   return VNET_API_ERROR_UNSPECIFIED;
1383                 }
1384             }
1385
1386           if (sm->num_workers > 1)
1387             {
1388               ip4_header_t ip = {
1389                 .src_address = local->addr,
1390               };
1391               tsm = vec_elt_at_index (sm->per_thread_data,
1392                                       sm->worker_in2out_cb (&ip, m->fib_index));
1393             }
1394           else
1395             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1396
1397           /* Delete sessions */
1398           u_key.addr = local->addr;
1399           u_key.fib_index = local->fib_index;
1400           kv.key = u_key.as_u64;
1401           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1402             {
1403               u = pool_elt_at_index (tsm->users, value.value);
1404               if (u->nstaticsessions)
1405                 {
1406                   head_index = u->sessions_per_user_list_head_index;
1407                   head = pool_elt_at_index (tsm->list_pool, head_index);
1408                   elt_index = head->next;
1409                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1410                   ses_index = elt->value;
1411                   while (ses_index != ~0)
1412                     {
1413                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1414                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1415                       ses_index = elt->value;
1416
1417                       if (!(is_lb_session (s)))
1418                         continue;
1419
1420                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1421                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1422                         continue;
1423
1424                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1425                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1426                     }
1427                 }
1428             }
1429       }));
1430       /* *INDENT-ON* */
1431       if (m->affinity)
1432         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1433       pool_free (m->locals);
1434       vec_free (m->tag);
1435       vec_free (m->workers);
1436
1437       pool_put (sm->static_mappings, m);
1438     }
1439
1440   return 0;
1441 }
1442
1443 int
1444 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1445                                        ip4_address_t l_addr, u16 l_port,
1446                                        snat_protocol_t proto, u32 vrf_id,
1447                                        u8 probability, u8 is_add)
1448 {
1449   snat_main_t *sm = &snat_main;
1450   snat_static_mapping_t *m = 0;
1451   snat_session_key_t m_key;
1452   clib_bihash_kv_8_8_t kv, value;
1453   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1454   snat_main_per_thread_data_t *tsm;
1455   snat_user_key_t u_key;
1456   snat_user_t *u;
1457   snat_session_t *s;
1458   dlist_elt_t *head, *elt;
1459   u32 elt_index, head_index, ses_index, *locals = 0;
1460   uword *bitmap = 0;
1461   int i;
1462
1463   if (!sm->endpoint_dependent)
1464     return VNET_API_ERROR_FEATURE_DISABLED;
1465
1466   m_key.addr = e_addr;
1467   m_key.port = e_port;
1468   m_key.protocol = proto;
1469   m_key.fib_index = 0;
1470   kv.key = m_key.as_u64;
1471   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1472     m = pool_elt_at_index (sm->static_mappings, value.value);
1473
1474   if (!m)
1475     return VNET_API_ERROR_NO_SUCH_ENTRY;
1476
1477   if (!is_lb_static_mapping (m))
1478     return VNET_API_ERROR_INVALID_VALUE;
1479
1480   /* *INDENT-OFF* */
1481   pool_foreach (local, m->locals,
1482   ({
1483     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1484         (local->vrf_id == vrf_id))
1485       {
1486         match_local = local;
1487         break;
1488       }
1489   }));
1490   /* *INDENT-ON* */
1491
1492   if (is_add)
1493     {
1494       if (match_local)
1495         return VNET_API_ERROR_VALUE_EXIST;
1496
1497       pool_get (m->locals, local);
1498       clib_memset (local, 0, sizeof (*local));
1499       local->addr.as_u32 = l_addr.as_u32;
1500       local->port = l_port;
1501       local->probability = probability;
1502       local->vrf_id = vrf_id;
1503       local->fib_index =
1504         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1505                                            FIB_SOURCE_PLUGIN_LOW);
1506
1507       if (!is_out2in_only_static_mapping (m))
1508         {
1509           m_key.addr = l_addr;
1510           m_key.port = l_port;
1511           m_key.fib_index = local->fib_index;
1512           kv.key = m_key.as_u64;
1513           kv.value = m - sm->static_mappings;
1514           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1515             nat_log_err ("static_mapping_by_local key add failed");
1516         }
1517     }
1518   else
1519     {
1520       if (!match_local)
1521         return VNET_API_ERROR_NO_SUCH_ENTRY;
1522
1523       if (pool_elts (m->locals) < 3)
1524         return VNET_API_ERROR_UNSPECIFIED;
1525
1526       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1527                         FIB_SOURCE_PLUGIN_LOW);
1528
1529       if (!is_out2in_only_static_mapping (m))
1530         {
1531           m_key.addr = l_addr;
1532           m_key.port = l_port;
1533           m_key.fib_index = match_local->fib_index;
1534           kv.key = m_key.as_u64;
1535           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1536             nat_log_err ("static_mapping_by_local key del failed");
1537         }
1538
1539       if (sm->num_workers > 1)
1540         {
1541           ip4_header_t ip = {
1542             .src_address = local->addr,
1543           };
1544           tsm = vec_elt_at_index (sm->per_thread_data,
1545                                   sm->worker_in2out_cb (&ip, m->fib_index));
1546         }
1547       else
1548         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1549
1550       /* Delete sessions */
1551       u_key.addr = match_local->addr;
1552       u_key.fib_index = match_local->fib_index;
1553       kv.key = u_key.as_u64;
1554       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1555         {
1556           u = pool_elt_at_index (tsm->users, value.value);
1557           if (u->nstaticsessions)
1558             {
1559               head_index = u->sessions_per_user_list_head_index;
1560               head = pool_elt_at_index (tsm->list_pool, head_index);
1561               elt_index = head->next;
1562               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1563               ses_index = elt->value;
1564               while (ses_index != ~0)
1565                 {
1566                   s = pool_elt_at_index (tsm->sessions, ses_index);
1567                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1568                   ses_index = elt->value;
1569
1570                   if (!(is_lb_session (s)))
1571                     continue;
1572
1573                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1574                       (clib_net_to_host_u16 (s->in2out.port) !=
1575                        match_local->port))
1576                     continue;
1577
1578                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1579                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1580                 }
1581             }
1582         }
1583
1584       pool_put (m->locals, match_local);
1585     }
1586
1587   vec_free (m->workers);
1588
1589   /* *INDENT-OFF* */
1590   pool_foreach (local, m->locals,
1591   ({
1592     vec_add1 (locals, local - m->locals);
1593     if (sm->num_workers > 1)
1594       {
1595         ip4_header_t ip;
1596         ip.src_address.as_u32 = local->addr.as_u32,
1597         bitmap = clib_bitmap_set (bitmap,
1598                                   sm->worker_in2out_cb (&ip, local->fib_index),
1599                                   1);
1600       }
1601   }));
1602   /* *INDENT-ON* */
1603
1604   ASSERT (vec_len (locals) > 1);
1605
1606   local = pool_elt_at_index (m->locals, locals[0]);
1607   local->prefix = local->probability;
1608   for (i = 1; i < vec_len (locals); i++)
1609     {
1610       local = pool_elt_at_index (m->locals, locals[i]);
1611       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1612       local->prefix = local->probability + prev_local->prefix;
1613     }
1614
1615   /* Assign workers */
1616   if (sm->num_workers > 1)
1617     {
1618       /* *INDENT-OFF* */
1619       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1620       /* *INDENT-ON* */
1621     }
1622
1623   return 0;
1624 }
1625
1626 int
1627 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1628                   u8 twice_nat)
1629 {
1630   snat_address_t *a = 0;
1631   snat_session_t *ses;
1632   u32 *ses_to_be_removed = 0, *ses_index;
1633   snat_main_per_thread_data_t *tsm;
1634   snat_static_mapping_t *m;
1635   snat_interface_t *interface;
1636   int i;
1637   snat_address_t *addresses =
1638     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1639
1640   /* Find SNAT address */
1641   for (i = 0; i < vec_len (addresses); i++)
1642     {
1643       if (addresses[i].addr.as_u32 == addr.as_u32)
1644         {
1645           a = addresses + i;
1646           break;
1647         }
1648     }
1649   if (!a)
1650     return VNET_API_ERROR_NO_SUCH_ENTRY;
1651
1652   if (delete_sm)
1653     {
1654       /* *INDENT-OFF* */
1655       pool_foreach (m, sm->static_mappings,
1656       ({
1657           if (m->external_addr.as_u32 == addr.as_u32)
1658             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1659                                             m->local_port, m->external_port,
1660                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1661                                             m->proto, 0, m->twice_nat,
1662                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1663       }));
1664       /* *INDENT-ON* */
1665     }
1666   else
1667     {
1668       /* Check if address is used in some static mapping */
1669       if (is_snat_address_used_in_static_mapping (sm, addr))
1670         {
1671           nat_log_notice ("address used in static mapping");
1672           return VNET_API_ERROR_UNSPECIFIED;
1673         }
1674     }
1675
1676   if (a->fib_index != ~0)
1677     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1678
1679   /* Delete sessions using address */
1680   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1681     {
1682       /* *INDENT-OFF* */
1683       vec_foreach (tsm, sm->per_thread_data)
1684         {
1685           pool_foreach (ses, tsm->sessions, ({
1686             if (ses->out2in.addr.as_u32 == addr.as_u32)
1687               {
1688                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1689                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1690               }
1691           }));
1692
1693           vec_foreach (ses_index, ses_to_be_removed)
1694             {
1695               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1696               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1697             }
1698
1699           vec_free (ses_to_be_removed);
1700         }
1701       /* *INDENT-ON* */
1702     }
1703
1704 #define _(N, i, n, s) \
1705   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1706   vec_free (a->busy_##n##_ports_per_thread);
1707   foreach_snat_protocol
1708 #undef _
1709     if (twice_nat)
1710     {
1711       vec_del1 (sm->twice_nat_addresses, i);
1712       return 0;
1713     }
1714   else
1715     vec_del1 (sm->addresses, i);
1716
1717   /* Delete external address from FIB */
1718   /* *INDENT-OFF* */
1719   pool_foreach (interface, sm->interfaces,
1720   ({
1721     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1722       continue;
1723
1724     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1725     break;
1726   }));
1727   pool_foreach (interface, sm->output_feature_interfaces,
1728   ({
1729     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1730       continue;
1731
1732     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1733     break;
1734   }));
1735   /* *INDENT-ON* */
1736
1737   return 0;
1738 }
1739
1740 int
1741 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1742 {
1743   snat_main_t *sm = &snat_main;
1744   snat_interface_t *i;
1745   const char *feature_name, *del_feature_name;
1746   snat_address_t *ap;
1747   snat_static_mapping_t *m;
1748   snat_det_map_t *dm;
1749   nat_outside_fib_t *outside_fib;
1750   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1751                                                        sw_if_index);
1752
1753   if (sm->out2in_dpo && !is_inside)
1754     return VNET_API_ERROR_UNSUPPORTED;
1755
1756   /* *INDENT-OFF* */
1757   pool_foreach (i, sm->output_feature_interfaces,
1758   ({
1759     if (i->sw_if_index == sw_if_index)
1760       return VNET_API_ERROR_VALUE_EXIST;
1761   }));
1762   /* *INDENT-ON* */
1763
1764   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1765     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1766   else
1767     {
1768       if (sm->num_workers > 1 && !sm->deterministic)
1769         feature_name =
1770           is_inside ? "nat44-in2out-worker-handoff" :
1771           "nat44-out2in-worker-handoff";
1772       else if (sm->deterministic)
1773         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1774       else if (sm->endpoint_dependent)
1775         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1776       else
1777         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1778     }
1779
1780   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1781     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1782                                                       NAT_FQ_NELTS);
1783
1784   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1785     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1786                                                       NAT_FQ_NELTS);
1787
1788   if (!is_inside)
1789     {
1790       /* *INDENT-OFF* */
1791       vec_foreach (outside_fib, sm->outside_fibs)
1792         {
1793           if (outside_fib->fib_index == fib_index)
1794             {
1795               if (is_del)
1796                 {
1797                   outside_fib->refcount--;
1798                   if (!outside_fib->refcount)
1799                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1800                 }
1801               else
1802                 outside_fib->refcount++;
1803               goto feature_set;
1804             }
1805         }
1806       /* *INDENT-ON* */
1807       if (!is_del)
1808         {
1809           vec_add2 (sm->outside_fibs, outside_fib, 1);
1810           outside_fib->refcount = 1;
1811           outside_fib->fib_index = fib_index;
1812         }
1813     }
1814 feature_set:
1815   /* *INDENT-OFF* */
1816   pool_foreach (i, sm->interfaces,
1817   ({
1818     if (i->sw_if_index == sw_if_index)
1819       {
1820         if (is_del)
1821           {
1822             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1823               {
1824                 if (is_inside)
1825                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1826                 else
1827                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1828
1829                 if (sm->num_workers > 1 && !sm->deterministic)
1830                   {
1831                     del_feature_name = "nat44-handoff-classify";
1832                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1833                                                  "nat44-out2in-worker-handoff";
1834                   }
1835                 else if (sm->deterministic)
1836                   {
1837                     del_feature_name = "nat44-det-classify";
1838                     feature_name = !is_inside ?  "nat44-det-in2out" :
1839                                                  "nat44-det-out2in";
1840                   }
1841                 else if (sm->endpoint_dependent)
1842                   {
1843                     del_feature_name = "nat44-ed-classify";
1844                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1845                                                  "nat44-ed-out2in";
1846                   }
1847                 else
1848                   {
1849                     del_feature_name = "nat44-classify";
1850                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1851                   }
1852
1853                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1854                                              sw_if_index, 0, 0, 0);
1855                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1856                                              sw_if_index, 1, 0, 0);
1857                 if (!is_inside)
1858                   {
1859                     if (sm->endpoint_dependent)
1860                       vnet_feature_enable_disable ("ip4-local",
1861                                                    "nat44-ed-hairpinning",
1862                                                    sw_if_index, 1, 0, 0);
1863                     else if (!sm->deterministic)
1864                       vnet_feature_enable_disable ("ip4-local",
1865                                                    "nat44-hairpinning",
1866                                                    sw_if_index, 1, 0, 0);
1867                   }
1868               }
1869             else
1870               {
1871                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1872                                              sw_if_index, 0, 0, 0);
1873                 pool_put (sm->interfaces, i);
1874                 if (is_inside)
1875                   {
1876                     if (sm->endpoint_dependent)
1877                       vnet_feature_enable_disable ("ip4-local",
1878                                                    "nat44-ed-hairpinning",
1879                                                    sw_if_index, 0, 0, 0);
1880                     else if (!sm->deterministic)
1881                       vnet_feature_enable_disable ("ip4-local",
1882                                                    "nat44-hairpinning",
1883                                                    sw_if_index, 0, 0, 0);
1884                   }
1885               }
1886           }
1887         else
1888           {
1889             if ((nat_interface_is_inside(i) && is_inside) ||
1890                 (nat_interface_is_outside(i) && !is_inside))
1891               return 0;
1892
1893             if (sm->num_workers > 1 && !sm->deterministic)
1894               {
1895                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1896                                                  "nat44-out2in-worker-handoff";
1897                 feature_name = "nat44-handoff-classify";
1898               }
1899             else if (sm->deterministic)
1900               {
1901                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1902                                                  "nat44-det-out2in";
1903                 feature_name = "nat44-det-classify";
1904               }
1905             else if (sm->endpoint_dependent)
1906               {
1907                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1908                                                  "nat44-ed-out2in";
1909                 feature_name = "nat44-ed-classify";
1910               }
1911             else
1912               {
1913                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1914                 feature_name = "nat44-classify";
1915               }
1916
1917             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1918                                          sw_if_index, 0, 0, 0);
1919             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1920                                          sw_if_index, 1, 0, 0);
1921             if (!is_inside)
1922               {
1923                 if (sm->endpoint_dependent)
1924                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1925                                                sw_if_index, 0, 0, 0);
1926                 else if (!sm->deterministic)
1927                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1928                                                sw_if_index, 0, 0, 0);
1929               }
1930             goto set_flags;
1931           }
1932
1933         goto fib;
1934       }
1935   }));
1936   /* *INDENT-ON* */
1937
1938   if (is_del)
1939     return VNET_API_ERROR_NO_SUCH_ENTRY;
1940
1941   pool_get (sm->interfaces, i);
1942   i->sw_if_index = sw_if_index;
1943   i->flags = 0;
1944   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1945                                0);
1946
1947   if (is_inside && !sm->out2in_dpo)
1948     {
1949       if (sm->endpoint_dependent)
1950         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1951                                      sw_if_index, 1, 0, 0);
1952       else if (!sm->deterministic)
1953         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1954                                      sw_if_index, 1, 0, 0);
1955     }
1956
1957 set_flags:
1958   if (is_inside)
1959     {
1960       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1961       return 0;
1962     }
1963   else
1964     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1965
1966   /* Add/delete external addresses to FIB */
1967 fib:
1968   /* *INDENT-OFF* */
1969   vec_foreach (ap, sm->addresses)
1970     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1971
1972   pool_foreach (m, sm->static_mappings,
1973   ({
1974     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1975       continue;
1976
1977     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1978   }));
1979
1980   pool_foreach (dm, sm->det_maps,
1981   ({
1982     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1983   }));
1984   /* *INDENT-ON* */
1985
1986   return 0;
1987 }
1988
1989 int
1990 snat_interface_add_del_output_feature (u32 sw_if_index,
1991                                        u8 is_inside, int is_del)
1992 {
1993   snat_main_t *sm = &snat_main;
1994   snat_interface_t *i;
1995   snat_address_t *ap;
1996   snat_static_mapping_t *m;
1997   nat_outside_fib_t *outside_fib;
1998   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1999                                                        sw_if_index);
2000
2001
2002   if (sm->deterministic ||
2003       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2004     return VNET_API_ERROR_UNSUPPORTED;
2005
2006   /* *INDENT-OFF* */
2007   pool_foreach (i, sm->interfaces,
2008   ({
2009     if (i->sw_if_index == sw_if_index)
2010       return VNET_API_ERROR_VALUE_EXIST;
2011   }));
2012   /* *INDENT-ON* */
2013
2014   if (!is_inside)
2015     {
2016       /* *INDENT-OFF* */
2017       vec_foreach (outside_fib, sm->outside_fibs)
2018         {
2019           if (outside_fib->fib_index == fib_index)
2020             {
2021               if (is_del)
2022                 {
2023                   outside_fib->refcount--;
2024                   if (!outside_fib->refcount)
2025                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2026                 }
2027               else
2028                 outside_fib->refcount++;
2029               goto feature_set;
2030             }
2031         }
2032       /* *INDENT-ON* */
2033       if (!is_del)
2034         {
2035           vec_add2 (sm->outside_fibs, outside_fib, 1);
2036           outside_fib->refcount = 1;
2037           outside_fib->fib_index = fib_index;
2038         }
2039     }
2040
2041 feature_set:
2042   if (is_inside)
2043     {
2044       if (sm->endpoint_dependent)
2045         {
2046           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2047                                        sw_if_index, !is_del, 0, 0);
2048           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2049                                        sw_if_index, !is_del, 0, 0);
2050         }
2051       else
2052         {
2053           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2054                                        sw_if_index, !is_del, 0, 0);
2055           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2056                                        sw_if_index, !is_del, 0, 0);
2057         }
2058       goto fq;
2059     }
2060
2061   if (sm->num_workers > 1)
2062     {
2063       vnet_feature_enable_disable ("ip4-unicast",
2064                                    "nat44-out2in-worker-handoff",
2065                                    sw_if_index, !is_del, 0, 0);
2066       vnet_feature_enable_disable ("ip4-output",
2067                                    "nat44-in2out-output-worker-handoff",
2068                                    sw_if_index, !is_del, 0, 0);
2069     }
2070   else
2071     {
2072       if (sm->endpoint_dependent)
2073         {
2074           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2075                                        sw_if_index, !is_del, 0, 0);
2076           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2077                                        sw_if_index, !is_del, 0, 0);
2078         }
2079       else
2080         {
2081           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2082                                        sw_if_index, !is_del, 0, 0);
2083           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2084                                        sw_if_index, !is_del, 0, 0);
2085         }
2086     }
2087
2088 fq:
2089   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2090     sm->fq_in2out_output_index =
2091       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2092
2093   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2094     sm->fq_out2in_index =
2095       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2096
2097   /* *INDENT-OFF* */
2098   pool_foreach (i, sm->output_feature_interfaces,
2099   ({
2100     if (i->sw_if_index == sw_if_index)
2101       {
2102         if (is_del)
2103           pool_put (sm->output_feature_interfaces, i);
2104         else
2105           return VNET_API_ERROR_VALUE_EXIST;
2106
2107         goto fib;
2108       }
2109   }));
2110   /* *INDENT-ON* */
2111
2112   if (is_del)
2113     return VNET_API_ERROR_NO_SUCH_ENTRY;
2114
2115   pool_get (sm->output_feature_interfaces, i);
2116   i->sw_if_index = sw_if_index;
2117   i->flags = 0;
2118   if (is_inside)
2119     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2120   else
2121     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2122
2123   /* Add/delete external addresses to FIB */
2124 fib:
2125   if (is_inside)
2126     return 0;
2127
2128   /* *INDENT-OFF* */
2129   vec_foreach (ap, sm->addresses)
2130     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2131
2132   pool_foreach (m, sm->static_mappings,
2133   ({
2134     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2135       continue;
2136
2137     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2138   }));
2139   /* *INDENT-ON* */
2140
2141   return 0;
2142 }
2143
2144 int
2145 snat_set_workers (uword * bitmap)
2146 {
2147   snat_main_t *sm = &snat_main;
2148   int i, j = 0;
2149
2150   if (sm->num_workers < 2)
2151     return VNET_API_ERROR_FEATURE_DISABLED;
2152
2153   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2154     return VNET_API_ERROR_INVALID_WORKER;
2155
2156   vec_free (sm->workers);
2157   /* *INDENT-OFF* */
2158   clib_bitmap_foreach (i, bitmap,
2159     ({
2160       vec_add1(sm->workers, i);
2161       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2162       j++;
2163     }));
2164   /* *INDENT-ON* */
2165
2166   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2167   sm->num_snat_thread = _vec_len (sm->workers);
2168
2169   return 0;
2170 }
2171
2172 static void
2173 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2174                          u32 old_fib_index)
2175 {
2176   snat_main_t *sm = &snat_main;
2177   nat_outside_fib_t *outside_fib;
2178   snat_interface_t *i;
2179   u8 is_add = 1;
2180
2181   if (new_fib_index == old_fib_index)
2182     return;
2183
2184   if (!vec_len (sm->outside_fibs))
2185     return;
2186
2187   pool_foreach (i, sm->interfaces, (
2188                                      {
2189                                      if (i->sw_if_index == sw_if_index)
2190                                      {
2191                                      if (!(nat_interface_is_outside (i)))
2192                                      return;}
2193                                      }
2194                 ));
2195   vec_foreach (outside_fib, sm->outside_fibs)
2196   {
2197     if (outside_fib->fib_index == old_fib_index)
2198       {
2199         outside_fib->refcount--;
2200         if (!outside_fib->refcount)
2201           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2202         break;
2203       }
2204   }
2205
2206   vec_foreach (outside_fib, sm->outside_fibs)
2207   {
2208     if (outside_fib->fib_index == new_fib_index)
2209       {
2210         outside_fib->refcount++;
2211         is_add = 0;
2212         break;
2213       }
2214   }
2215
2216   if (is_add)
2217     {
2218       vec_add2 (sm->outside_fibs, outside_fib, 1);
2219       outside_fib->refcount = 1;
2220       outside_fib->fib_index = new_fib_index;
2221     }
2222 }
2223
2224 static void
2225 snat_ip4_table_bind (ip4_main_t * im,
2226                      uword opaque,
2227                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2228 {
2229   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2230 }
2231
2232 static void
2233 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2234                                        uword opaque,
2235                                        u32 sw_if_index,
2236                                        ip4_address_t * address,
2237                                        u32 address_length,
2238                                        u32 if_address_index, u32 is_delete);
2239
2240 static void
2241 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2242                                  uword opaque,
2243                                  u32 sw_if_index,
2244                                  ip4_address_t * address,
2245                                  u32 address_length,
2246                                  u32 if_address_index, u32 is_delete);
2247
2248 static int
2249 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2250                                  u32 fib_index,
2251                                  u32 thread_index,
2252                                  snat_session_key_t * k,
2253                                  u16 port_per_thread, u32 snat_thread_index);
2254
2255 static clib_error_t *
2256 snat_init (vlib_main_t * vm)
2257 {
2258   snat_main_t *sm = &snat_main;
2259   clib_error_t *error = 0;
2260   ip4_main_t *im = &ip4_main;
2261   ip_lookup_main_t *lm = &im->lookup_main;
2262   uword *p;
2263   vlib_thread_registration_t *tr;
2264   vlib_thread_main_t *tm = vlib_get_thread_main ();
2265   uword *bitmap = 0;
2266   u32 i;
2267   ip4_add_del_interface_address_callback_t cb4;
2268   vlib_node_t *node;
2269
2270   sm->vlib_main = vm;
2271   sm->vnet_main = vnet_get_main ();
2272   sm->ip4_main = im;
2273   sm->ip4_lookup_main = lm;
2274   sm->api_main = &api_main;
2275   sm->first_worker_index = 0;
2276   sm->num_workers = 0;
2277   sm->num_snat_thread = 1;
2278   sm->workers = 0;
2279   sm->port_per_thread = 0xffff - 1024;
2280   sm->fq_in2out_index = ~0;
2281   sm->fq_in2out_output_index = ~0;
2282   sm->fq_out2in_index = ~0;
2283   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2284   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2285   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2286   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2287   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2288   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2289   sm->forwarding_enabled = 0;
2290   sm->log_class = vlib_log_register_class ("nat", 0);
2291   sm->mss_clamping = 0;
2292
2293   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2294   sm->error_node_index = node->index;
2295
2296   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2297   sm->in2out_node_index = node->index;
2298   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2299   sm->in2out_output_node_index = node->index;
2300   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2301   sm->in2out_fast_node_index = node->index;
2302   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2303   sm->in2out_slowpath_node_index = node->index;
2304   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2305   sm->in2out_slowpath_output_node_index = node->index;
2306   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-reass");
2307   sm->in2out_reass_node_index = node->index;
2308
2309   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2310   sm->ed_in2out_node_index = node->index;
2311   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2312   sm->ed_in2out_slowpath_node_index = node->index;
2313   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-reass");
2314   sm->ed_in2out_reass_node_index = node->index;
2315
2316   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2317   sm->out2in_node_index = node->index;
2318   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2319   sm->out2in_fast_node_index = node->index;
2320   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-reass");
2321   sm->out2in_reass_node_index = node->index;
2322
2323   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2324   sm->ed_out2in_node_index = node->index;
2325   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2326   sm->ed_out2in_slowpath_node_index = node->index;
2327   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-reass");
2328   sm->ed_out2in_reass_node_index = node->index;
2329
2330   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2331   sm->det_in2out_node_index = node->index;
2332   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2333   sm->det_out2in_node_index = node->index;
2334
2335   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2336   sm->hairpinning_node_index = node->index;
2337   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2338   sm->hairpin_dst_node_index = node->index;
2339   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2340   sm->hairpin_src_node_index = node->index;
2341   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2342   sm->ed_hairpinning_node_index = node->index;
2343   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2344   sm->ed_hairpin_dst_node_index = node->index;
2345   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2346   sm->ed_hairpin_src_node_index = node->index;
2347
2348   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2349   if (p)
2350     {
2351       tr = (vlib_thread_registration_t *) p[0];
2352       if (tr)
2353         {
2354           sm->num_workers = tr->count;
2355           sm->first_worker_index = tr->first_index;
2356         }
2357     }
2358
2359   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2360
2361   /* Use all available workers by default */
2362   if (sm->num_workers > 1)
2363     {
2364       for (i = 0; i < sm->num_workers; i++)
2365         bitmap = clib_bitmap_set (bitmap, i, 1);
2366       snat_set_workers (bitmap);
2367       clib_bitmap_free (bitmap);
2368     }
2369   else
2370     {
2371       sm->per_thread_data[0].snat_thread_index = 0;
2372     }
2373
2374   error = snat_api_init (vm, sm);
2375   if (error)
2376     return error;
2377
2378   /* Set up the interface address add/del callback */
2379   cb4.function = snat_ip4_add_del_interface_address_cb;
2380   cb4.function_opaque = 0;
2381
2382   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2383
2384   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2385   cb4.function_opaque = 0;
2386
2387   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2388
2389   nat_dpo_module_init ();
2390
2391   /* Init counters */
2392   sm->total_users.name = "total-users";
2393   sm->total_users.stat_segment_name = "/nat44/total-users";
2394   vlib_validate_simple_counter (&sm->total_users, 0);
2395   vlib_zero_simple_counter (&sm->total_users, 0);
2396   sm->total_sessions.name = "total-sessions";
2397   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2398   vlib_validate_simple_counter (&sm->total_sessions, 0);
2399   vlib_zero_simple_counter (&sm->total_sessions, 0);
2400
2401   /* Init IPFIX logging */
2402   snat_ipfix_logging_init (vm);
2403
2404   /* Init NAT64 */
2405   error = nat64_init (vm);
2406   if (error)
2407     return error;
2408
2409   dslite_init (vm);
2410
2411   nat66_init (vm);
2412
2413   ip4_table_bind_callback_t cbt4 = {
2414     .function = snat_ip4_table_bind,
2415   };
2416   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2417
2418   /* Init virtual fragmenentation reassembly */
2419   return nat_reass_init (vm);
2420 }
2421
2422 VLIB_INIT_FUNCTION (snat_init);
2423
2424 void
2425 snat_free_outside_address_and_port (snat_address_t * addresses,
2426                                     u32 thread_index, snat_session_key_t * k)
2427 {
2428   snat_address_t *a;
2429   u32 address_index;
2430   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2431
2432   for (address_index = 0; address_index < vec_len (addresses);
2433        address_index++)
2434     {
2435       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2436         break;
2437     }
2438
2439   ASSERT (address_index < vec_len (addresses));
2440
2441   a = addresses + address_index;
2442
2443   switch (k->protocol)
2444     {
2445 #define _(N, i, n, s) \
2446     case SNAT_PROTOCOL_##N: \
2447       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2448         port_host_byte_order) == 1); \
2449       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2450         port_host_byte_order, 0); \
2451       a->busy_##n##_ports--; \
2452       a->busy_##n##_ports_per_thread[thread_index]--; \
2453       break;
2454       foreach_snat_protocol
2455 #undef _
2456     default:
2457       nat_log_info ("unknown protocol");
2458       return;
2459     }
2460 }
2461
2462 static int
2463 nat_set_outside_address_and_port (snat_address_t * addresses,
2464                                   u32 thread_index, snat_session_key_t * k)
2465 {
2466   snat_address_t *a = 0;
2467   u32 address_index;
2468   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2469
2470   for (address_index = 0; address_index < vec_len (addresses);
2471        address_index++)
2472     {
2473       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2474         continue;
2475
2476       a = addresses + address_index;
2477       switch (k->protocol)
2478         {
2479 #define _(N, j, n, s) \
2480         case SNAT_PROTOCOL_##N: \
2481           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2482             return VNET_API_ERROR_INSTANCE_IN_USE; \
2483           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2484           a->busy_##n##_ports_per_thread[thread_index]++; \
2485           a->busy_##n##_ports++; \
2486           return 0;
2487           foreach_snat_protocol
2488 #undef _
2489         default:
2490           nat_log_info ("unknown protocol");
2491           return 1;
2492         }
2493     }
2494
2495   return VNET_API_ERROR_NO_SUCH_ENTRY;
2496 }
2497
2498 int
2499 snat_static_mapping_match (snat_main_t * sm,
2500                            snat_session_key_t match,
2501                            snat_session_key_t * mapping,
2502                            u8 by_external,
2503                            u8 * is_addr_only,
2504                            twice_nat_type_t * twice_nat,
2505                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2506                            u8 * is_identity_nat)
2507 {
2508   clib_bihash_kv_8_8_t kv, value;
2509   snat_static_mapping_t *m;
2510   snat_session_key_t m_key;
2511   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2512   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2513   u8 backend_index;
2514   nat44_lb_addr_port_t *local;
2515
2516   m_key.fib_index = match.fib_index;
2517   if (by_external)
2518     {
2519       mapping_hash = &sm->static_mapping_by_external;
2520       m_key.fib_index = 0;
2521     }
2522
2523   m_key.addr = match.addr;
2524   m_key.port = clib_net_to_host_u16 (match.port);
2525   m_key.protocol = match.protocol;
2526
2527   kv.key = m_key.as_u64;
2528
2529   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2530     {
2531       /* Try address only mapping */
2532       m_key.port = 0;
2533       m_key.protocol = 0;
2534       kv.key = m_key.as_u64;
2535       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2536         return 1;
2537     }
2538
2539   m = pool_elt_at_index (sm->static_mappings, value.value);
2540
2541   if (by_external)
2542     {
2543       if (is_lb_static_mapping (m))
2544         {
2545           if (PREDICT_FALSE (lb != 0))
2546             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2547           if (m->affinity)
2548             {
2549               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2550                                               match.protocol, match.port,
2551                                               &backend_index))
2552                 goto get_local;
2553
2554               local = pool_elt_at_index (m->locals, backend_index);
2555               mapping->addr = local->addr;
2556               mapping->port = clib_host_to_net_u16 (local->port);
2557               mapping->fib_index = local->fib_index;
2558               goto end;
2559             }
2560         get_local:
2561           /* *INDENT-OFF* */
2562           pool_foreach_index (i, m->locals,
2563           ({
2564             vec_add1 (tmp, i);
2565           }));
2566           /* *INDENT-ON* */
2567           hi = vec_len (tmp) - 1;
2568           local = pool_elt_at_index (m->locals, tmp[hi]);
2569           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2570           while (lo < hi)
2571             {
2572               mid = ((hi - lo) >> 1) + lo;
2573               local = pool_elt_at_index (m->locals, tmp[mid]);
2574               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2575             }
2576           local = pool_elt_at_index (m->locals, tmp[lo]);
2577           if (!(local->prefix >= rand))
2578             return 1;
2579           if (PREDICT_FALSE (sm->num_workers > 1))
2580             {
2581               ip4_header_t ip = {
2582                 .src_address = local->addr,
2583               };
2584               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2585                   vlib_get_thread_index ())
2586                 goto get_local;
2587             }
2588           mapping->addr = local->addr;
2589           mapping->port = clib_host_to_net_u16 (local->port);
2590           mapping->fib_index = local->fib_index;
2591           if (m->affinity)
2592             {
2593               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2594                                                 match.protocol, match.port,
2595                                                 tmp[lo], m->affinity,
2596                                                 m->affinity_per_service_list_head_index))
2597                 nat_log_info ("create affinity record failed");
2598             }
2599           vec_free (tmp);
2600         }
2601       else
2602         {
2603           if (PREDICT_FALSE (lb != 0))
2604             *lb = NO_LB_NAT;
2605           mapping->fib_index = m->fib_index;
2606           mapping->addr = m->local_addr;
2607           /* Address only mapping doesn't change port */
2608           mapping->port = is_addr_only_static_mapping (m) ? match.port
2609             : clib_host_to_net_u16 (m->local_port);
2610         }
2611       mapping->protocol = m->proto;
2612     }
2613   else
2614     {
2615       mapping->addr = m->external_addr;
2616       /* Address only mapping doesn't change port */
2617       mapping->port = is_addr_only_static_mapping (m) ? match.port
2618         : clib_host_to_net_u16 (m->external_port);
2619       mapping->fib_index = sm->outside_fib_index;
2620     }
2621
2622 end:
2623   if (PREDICT_FALSE (is_addr_only != 0))
2624     *is_addr_only = is_addr_only_static_mapping (m);
2625
2626   if (PREDICT_FALSE (twice_nat != 0))
2627     *twice_nat = m->twice_nat;
2628
2629   if (PREDICT_FALSE (is_identity_nat != 0))
2630     *is_identity_nat = is_identity_static_mapping (m);
2631
2632   return 0;
2633 }
2634
2635 static_always_inline u16
2636 snat_random_port (u16 min, u16 max)
2637 {
2638   snat_main_t *sm = &snat_main;
2639   return min + random_u32 (&sm->random_seed) /
2640     (random_u32_max () / (max - min + 1) + 1);
2641 }
2642
2643 int
2644 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2645                                      u32 fib_index,
2646                                      u32 thread_index,
2647                                      snat_session_key_t * k,
2648                                      u16 port_per_thread,
2649                                      u32 snat_thread_index)
2650 {
2651   snat_main_t *sm = &snat_main;
2652
2653   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2654                                   port_per_thread, snat_thread_index);
2655 }
2656
2657 static int
2658 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2659                                  u32 fib_index,
2660                                  u32 thread_index,
2661                                  snat_session_key_t * k,
2662                                  u16 port_per_thread, u32 snat_thread_index)
2663 {
2664   int i;
2665   snat_address_t *a, *ga = 0;
2666   u32 portnum;
2667
2668   for (i = 0; i < vec_len (addresses); i++)
2669     {
2670       a = addresses + i;
2671       switch (k->protocol)
2672         {
2673 #define _(N, j, n, s) \
2674         case SNAT_PROTOCOL_##N: \
2675           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2676             { \
2677               if (a->fib_index == fib_index) \
2678                 { \
2679                   while (1) \
2680                     { \
2681                       portnum = (port_per_thread * \
2682                         snat_thread_index) + \
2683                         snat_random_port(1, port_per_thread) + 1024; \
2684                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2685                         continue; \
2686                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2687                       a->busy_##n##_ports_per_thread[thread_index]++; \
2688                       a->busy_##n##_ports++; \
2689                       k->addr = a->addr; \
2690                       k->port = clib_host_to_net_u16(portnum); \
2691                       return 0; \
2692                     } \
2693                 } \
2694               else if (a->fib_index == ~0) \
2695                 { \
2696                   ga = a; \
2697                 } \
2698             } \
2699           break;
2700           foreach_snat_protocol
2701 #undef _
2702         default:
2703           nat_log_info ("unknown protocol");
2704           return 1;
2705         }
2706
2707     }
2708
2709   if (ga)
2710     {
2711       a = ga;
2712       switch (k->protocol)
2713         {
2714 #define _(N, j, n, s) \
2715         case SNAT_PROTOCOL_##N: \
2716           while (1) \
2717             { \
2718               portnum = (port_per_thread * \
2719                 snat_thread_index) + \
2720                 snat_random_port(1, port_per_thread) + 1024; \
2721               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2722                 continue; \
2723               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2724               a->busy_##n##_ports_per_thread[thread_index]++; \
2725               a->busy_##n##_ports++; \
2726               k->addr = a->addr; \
2727               k->port = clib_host_to_net_u16(portnum); \
2728               return 0; \
2729             }
2730           break;
2731           foreach_snat_protocol
2732 #undef _
2733         default:
2734           nat_log_info ("unknown protocol");
2735           return 1;
2736         }
2737     }
2738
2739   /* Totally out of translations to use... */
2740   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2741   return 1;
2742 }
2743
2744 static int
2745 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2746                               u32 fib_index,
2747                               u32 thread_index,
2748                               snat_session_key_t * k,
2749                               u16 port_per_thread, u32 snat_thread_index)
2750 {
2751   snat_main_t *sm = &snat_main;
2752   snat_address_t *a = addresses;
2753   u16 m, ports, portnum, A, j;
2754   m = 16 - (sm->psid_offset + sm->psid_length);
2755   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2756
2757   if (!vec_len (addresses))
2758     goto exhausted;
2759
2760   switch (k->protocol)
2761     {
2762 #define _(N, i, n, s) \
2763     case SNAT_PROTOCOL_##N: \
2764       if (a->busy_##n##_ports < ports) \
2765         { \
2766           while (1) \
2767             { \
2768               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2769               j = snat_random_port(0, pow2_mask(m)); \
2770               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2771               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2772                 continue; \
2773               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2774               a->busy_##n##_ports++; \
2775               k->addr = a->addr; \
2776               k->port = clib_host_to_net_u16 (portnum); \
2777               return 0; \
2778             } \
2779         } \
2780       break;
2781       foreach_snat_protocol
2782 #undef _
2783     default:
2784       nat_log_info ("unknown protocol");
2785       return 1;
2786     }
2787
2788 exhausted:
2789   /* Totally out of translations to use... */
2790   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2791   return 1;
2792 }
2793
2794 static int
2795 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2796                                u32 fib_index,
2797                                u32 thread_index,
2798                                snat_session_key_t * k,
2799                                u16 port_per_thread, u32 snat_thread_index)
2800 {
2801   snat_main_t *sm = &snat_main;
2802   snat_address_t *a = addresses;
2803   u16 portnum, ports;
2804
2805   ports = sm->end_port - sm->start_port + 1;
2806
2807   if (!vec_len (addresses))
2808     goto exhausted;
2809
2810   switch (k->protocol)
2811     {
2812 #define _(N, i, n, s) \
2813     case SNAT_PROTOCOL_##N: \
2814       if (a->busy_##n##_ports < ports) \
2815         { \
2816           while (1) \
2817             { \
2818               portnum = snat_random_port(sm->start_port, sm->end_port); \
2819               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2820                 continue; \
2821               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2822               a->busy_##n##_ports++; \
2823               k->addr = a->addr; \
2824               k->port = clib_host_to_net_u16 (portnum); \
2825               return 0; \
2826             } \
2827         } \
2828       break;
2829       foreach_snat_protocol
2830 #undef _
2831     default:
2832       nat_log_info ("unknown protocol");
2833       return 1;
2834     }
2835
2836 exhausted:
2837   /* Totally out of translations to use... */
2838   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2839   return 1;
2840 }
2841
2842 void
2843 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2844 {
2845   dpo_id_t dpo_v4 = DPO_INVALID;
2846   fib_prefix_t pfx = {
2847     .fp_proto = FIB_PROTOCOL_IP4,
2848     .fp_len = 32,
2849     .fp_addr.ip4.as_u32 = addr.as_u32,
2850   };
2851
2852   if (is_add)
2853     {
2854       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2855       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2856                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2857       dpo_reset (&dpo_v4);
2858     }
2859   else
2860     {
2861       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2862     }
2863 }
2864
2865 u8 *
2866 format_session_kvp (u8 * s, va_list * args)
2867 {
2868   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2869   snat_session_key_t k;
2870
2871   k.as_u64 = v->key;
2872
2873   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2874
2875   return s;
2876 }
2877
2878 u8 *
2879 format_static_mapping_kvp (u8 * s, va_list * args)
2880 {
2881   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2882   snat_session_key_t k;
2883
2884   k.as_u64 = v->key;
2885
2886   s = format (s, "%U static-mapping-index %llu",
2887               format_static_mapping_key, &k, v->value);
2888
2889   return s;
2890 }
2891
2892 u8 *
2893 format_user_kvp (u8 * s, va_list * args)
2894 {
2895   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2896   snat_user_key_t k;
2897
2898   k.as_u64 = v->key;
2899
2900   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2901               k.fib_index, v->value);
2902
2903   return s;
2904 }
2905
2906 u8 *
2907 format_ed_session_kvp (u8 * s, va_list * args)
2908 {
2909   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2910   nat_ed_ses_key_t k;
2911
2912   k.as_u64[0] = v->key[0];
2913   k.as_u64[1] = v->key[1];
2914
2915   s =
2916     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2917             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2918             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2919             format_ip_protocol, k.proto, k.fib_index, v->value);
2920
2921   return s;
2922 }
2923
2924 static u32
2925 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2926 {
2927   snat_main_t *sm = &snat_main;
2928   u32 next_worker_index = 0;
2929   u32 hash;
2930
2931   next_worker_index = sm->first_worker_index;
2932   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2933     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2934
2935   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2936     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2937   else
2938     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2939
2940   return next_worker_index;
2941 }
2942
2943 static u32
2944 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2945 {
2946   snat_main_t *sm = &snat_main;
2947   udp_header_t *udp;
2948   u16 port;
2949   snat_session_key_t m_key;
2950   clib_bihash_kv_8_8_t kv, value;
2951   snat_static_mapping_t *m;
2952   u32 proto;
2953   u32 next_worker_index = 0;
2954
2955   /* first try static mappings without port */
2956   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2957     {
2958       m_key.addr = ip0->dst_address;
2959       m_key.port = 0;
2960       m_key.protocol = 0;
2961       m_key.fib_index = rx_fib_index0;
2962       kv.key = m_key.as_u64;
2963       if (!clib_bihash_search_8_8
2964           (&sm->static_mapping_by_external, &kv, &value))
2965         {
2966           m = pool_elt_at_index (sm->static_mappings, value.value);
2967           return m->workers[0];
2968         }
2969     }
2970
2971   proto = ip_proto_to_snat_proto (ip0->protocol);
2972   udp = ip4_next_header (ip0);
2973   port = udp->dst_port;
2974
2975   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2976     {
2977       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2978         return vlib_get_thread_index ();
2979
2980       nat_reass_ip4_t *reass;
2981       reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2982                                   ip0->fragment_id, ip0->protocol);
2983
2984       if (reass && (reass->thread_index != (u32) ~ 0))
2985         return reass->thread_index;
2986
2987       if (ip4_is_first_fragment (ip0))
2988         {
2989           reass =
2990             nat_ip4_reass_create (ip0->src_address, ip0->dst_address,
2991                                   ip0->fragment_id, ip0->protocol);
2992           if (!reass)
2993             goto no_reass;
2994
2995           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2996             {
2997               m_key.addr = ip0->dst_address;
2998               m_key.port = clib_net_to_host_u16 (port);
2999               m_key.protocol = proto;
3000               m_key.fib_index = rx_fib_index0;
3001               kv.key = m_key.as_u64;
3002               if (!clib_bihash_search_8_8
3003                   (&sm->static_mapping_by_external, &kv, &value))
3004                 {
3005                   m = pool_elt_at_index (sm->static_mappings, value.value);
3006                   reass->thread_index = m->workers[0];
3007                   return reass->thread_index;
3008                 }
3009             }
3010           reass->thread_index = sm->first_worker_index;
3011           reass->thread_index +=
3012             sm->workers[(clib_net_to_host_u16 (port) - 1024) /
3013                         sm->port_per_thread];
3014           return reass->thread_index;
3015         }
3016       else
3017         return vlib_get_thread_index ();
3018     }
3019
3020 no_reass:
3021   /* unknown protocol */
3022   if (PREDICT_FALSE (proto == ~0))
3023     {
3024       /* use current thread */
3025       return vlib_get_thread_index ();
3026     }
3027
3028   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3029     {
3030       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3031       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3032       if (!icmp_is_error_message (icmp))
3033         port = echo->identifier;
3034       else
3035         {
3036           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3037           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3038           void *l4_header = ip4_next_header (inner_ip);
3039           switch (proto)
3040             {
3041             case SNAT_PROTOCOL_ICMP:
3042               icmp = (icmp46_header_t *) l4_header;
3043               echo = (icmp_echo_header_t *) (icmp + 1);
3044               port = echo->identifier;
3045               break;
3046             case SNAT_PROTOCOL_UDP:
3047             case SNAT_PROTOCOL_TCP:
3048               port = ((tcp_udp_header_t *) l4_header)->src_port;
3049               break;
3050             default:
3051               return vlib_get_thread_index ();
3052             }
3053         }
3054     }
3055
3056   /* try static mappings with port */
3057   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3058     {
3059       m_key.addr = ip0->dst_address;
3060       m_key.port = clib_net_to_host_u16 (port);
3061       m_key.protocol = proto;
3062       m_key.fib_index = rx_fib_index0;
3063       kv.key = m_key.as_u64;
3064       if (!clib_bihash_search_8_8
3065           (&sm->static_mapping_by_external, &kv, &value))
3066         {
3067           m = pool_elt_at_index (sm->static_mappings, value.value);
3068           return m->workers[0];
3069         }
3070     }
3071
3072   /* worker by outside port */
3073   next_worker_index = sm->first_worker_index;
3074   next_worker_index +=
3075     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3076   return next_worker_index;
3077 }
3078
3079 static u32
3080 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
3081 {
3082   snat_main_t *sm = &snat_main;
3083   clib_bihash_kv_8_8_t kv, value;
3084   u32 proto, next_worker_index = 0;
3085   udp_header_t *udp;
3086   u16 port;
3087   snat_static_mapping_t *m;
3088   u32 hash;
3089
3090   /* first try static mappings without port */
3091   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3092     {
3093       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
3094       if (!clib_bihash_search_8_8
3095           (&sm->static_mapping_by_external, &kv, &value))
3096         {
3097           m = pool_elt_at_index (sm->static_mappings, value.value);
3098           return m->workers[0];
3099         }
3100     }
3101
3102   proto = ip_proto_to_snat_proto (ip->protocol);
3103
3104   /* unknown protocol */
3105   if (PREDICT_FALSE (proto == ~0))
3106     {
3107       /* use current thread */
3108       return vlib_get_thread_index ();
3109     }
3110
3111   udp = ip4_next_header (ip);
3112   port = udp->dst_port;
3113
3114   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3115     {
3116       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3117       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3118       if (!icmp_is_error_message (icmp))
3119         port = echo->identifier;
3120       else
3121         {
3122           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3123           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3124           void *l4_header = ip4_next_header (inner_ip);
3125           switch (proto)
3126             {
3127             case SNAT_PROTOCOL_ICMP:
3128               icmp = (icmp46_header_t *) l4_header;
3129               echo = (icmp_echo_header_t *) (icmp + 1);
3130               port = echo->identifier;
3131               break;
3132             case SNAT_PROTOCOL_UDP:
3133             case SNAT_PROTOCOL_TCP:
3134               port = ((tcp_udp_header_t *) l4_header)->src_port;
3135               break;
3136             default:
3137               return vlib_get_thread_index ();
3138             }
3139         }
3140     }
3141
3142   /* try static mappings with port */
3143   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3144     {
3145       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
3146                   clib_net_to_host_u16 (port));
3147       if (!clib_bihash_search_8_8
3148           (&sm->static_mapping_by_external, &kv, &value))
3149         {
3150           m = pool_elt_at_index (sm->static_mappings, value.value);
3151           if (!is_lb_static_mapping (m))
3152             return m->workers[0];
3153
3154           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3155             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3156
3157           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3158             return m->workers[hash & (_vec_len (m->workers) - 1)];
3159           else
3160             return m->workers[hash % _vec_len (m->workers)];
3161         }
3162     }
3163
3164   /* worker by outside port */
3165   next_worker_index = sm->first_worker_index;
3166   next_worker_index +=
3167     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3168
3169   return next_worker_index;
3170 }
3171
3172 void
3173 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3174                 ip4_address_t * out_addr, u16 out_port,
3175                 ip4_address_t * eh_addr, u16 eh_port,
3176                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3177                 u32 fib_index, u16 flags, u32 thread_index)
3178 {
3179   snat_main_t *sm = &snat_main;
3180   snat_session_key_t key;
3181   snat_user_t *u;
3182   snat_session_t *s;
3183   clib_bihash_kv_8_8_t kv;
3184   f64 now = vlib_time_now (sm->vlib_main);
3185   nat_outside_fib_t *outside_fib;
3186   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3187   snat_main_per_thread_data_t *tsm;
3188   fib_prefix_t pfx = {
3189     .fp_proto = FIB_PROTOCOL_IP4,
3190     .fp_len = 32,
3191     .fp_addr = {
3192                 .ip4.as_u32 = eh_addr->as_u32,
3193                 },
3194   };
3195
3196   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3197
3198   key.addr.as_u32 = out_addr->as_u32;
3199   key.port = out_port;
3200   key.protocol = proto;
3201
3202   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3203     {
3204       if (nat_set_outside_address_and_port
3205           (sm->addresses, thread_index, &key))
3206         return;
3207     }
3208
3209   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3210   if (!u)
3211     return;
3212
3213   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3214   if (!s)
3215     return;
3216
3217   s->last_heard = now;
3218   s->flags = flags;
3219   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3220   s->ext_host_port = eh_port;
3221   user_session_increment (sm, u, snat_is_session_static (s));
3222   switch (vec_len (sm->outside_fibs))
3223     {
3224     case 0:
3225       key.fib_index = sm->outside_fib_index;
3226       break;
3227     case 1:
3228       key.fib_index = sm->outside_fibs[0].fib_index;
3229       break;
3230     default:
3231       /* *INDENT-OFF* */
3232       vec_foreach (outside_fib, sm->outside_fibs)
3233         {
3234           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3235           if (FIB_NODE_INDEX_INVALID != fei)
3236             {
3237               if (fib_entry_get_resolving_interface (fei) != ~0)
3238                 {
3239                   key.fib_index = outside_fib->fib_index;
3240                   break;
3241                 }
3242             }
3243         }
3244       /* *INDENT-ON* */
3245       break;
3246     }
3247   s->out2in = key;
3248   kv.key = key.as_u64;
3249   kv.value = s - tsm->sessions;
3250   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3251     nat_log_warn ("out2in key add failed");
3252
3253   key.addr.as_u32 = in_addr->as_u32;
3254   key.port = in_port;
3255   key.fib_index = fib_index;
3256   s->in2out = key;
3257   kv.key = key.as_u64;
3258   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3259     nat_log_warn ("in2out key add failed");
3260 }
3261
3262 void
3263 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3264                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3265                 u32 ti)
3266 {
3267   snat_main_t *sm = &snat_main;
3268   snat_session_key_t key;
3269   clib_bihash_kv_8_8_t kv, value;
3270   u32 thread_index;
3271   snat_session_t *s;
3272   snat_main_per_thread_data_t *tsm;
3273
3274   if (sm->num_workers > 1)
3275     thread_index =
3276       sm->first_worker_index +
3277       (sm->workers[(clib_net_to_host_u16 (out_port) -
3278                     1024) / sm->port_per_thread]);
3279   else
3280     thread_index = sm->num_workers;
3281   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3282
3283   key.addr.as_u32 = out_addr->as_u32;
3284   key.port = out_port;
3285   key.protocol = proto;
3286   key.fib_index = fib_index;
3287   kv.key = key.as_u64;
3288   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3289     return;
3290
3291   s = pool_elt_at_index (tsm->sessions, value.value);
3292   nat_free_session_data (sm, s, thread_index, 1);
3293   nat44_delete_session (sm, s, thread_index);
3294 }
3295
3296 void
3297 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3298                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3299                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3300 {
3301   snat_main_t *sm = &snat_main;
3302   snat_session_key_t key;
3303   clib_bihash_kv_8_8_t kv, value;
3304   snat_session_t *s;
3305   snat_main_per_thread_data_t *tsm;
3306
3307   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3308
3309   key.addr.as_u32 = out_addr->as_u32;
3310   key.port = out_port;
3311   key.protocol = proto;
3312   key.fib_index = fib_index;
3313   kv.key = key.as_u64;
3314   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3315     return;
3316
3317   s = pool_elt_at_index (tsm->sessions, value.value);
3318   s->total_pkts = total_pkts;
3319   s->total_bytes = total_bytes;
3320 }
3321
3322 void
3323 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3324                    ip4_address_t * out_addr, u16 out_port,
3325                    ip4_address_t * eh_addr, u16 eh_port,
3326                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3327                    u32 fib_index, u16 flags, u32 thread_index)
3328 {
3329   snat_main_t *sm = &snat_main;
3330   snat_session_key_t key;
3331   snat_user_t *u;
3332   snat_session_t *s;
3333   clib_bihash_kv_16_8_t kv;
3334   f64 now = vlib_time_now (sm->vlib_main);
3335   nat_outside_fib_t *outside_fib;
3336   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3337   snat_main_per_thread_data_t *tsm;
3338   fib_prefix_t pfx = {
3339     .fp_proto = FIB_PROTOCOL_IP4,
3340     .fp_len = 32,
3341     .fp_addr = {
3342                 .ip4.as_u32 = eh_addr->as_u32,
3343                 },
3344   };
3345
3346   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3347
3348   key.addr.as_u32 = out_addr->as_u32;
3349   key.port = out_port;
3350   key.protocol = proto;
3351
3352   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3353     {
3354       if (nat_set_outside_address_and_port
3355           (sm->addresses, thread_index, &key))
3356         return;
3357     }
3358
3359   key.addr.as_u32 = ehn_addr->as_u32;
3360   key.port = ehn_port;
3361   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3362     {
3363       if (nat_set_outside_address_and_port
3364           (sm->twice_nat_addresses, thread_index, &key))
3365         return;
3366     }
3367
3368   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3369   if (!u)
3370     return;
3371
3372   s = nat_ed_session_alloc (sm, u, thread_index, now);
3373   if (!s)
3374     return;
3375
3376   s->last_heard = now;
3377   s->flags = flags;
3378   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3379   s->ext_host_nat_port = s->ext_host_port = eh_port;
3380   if (is_twice_nat_session (s))
3381     {
3382       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3383       s->ext_host_nat_port = ehn_port;
3384     }
3385   user_session_increment (sm, u, snat_is_session_static (s));
3386   switch (vec_len (sm->outside_fibs))
3387     {
3388     case 0:
3389       key.fib_index = sm->outside_fib_index;
3390       break;
3391     case 1:
3392       key.fib_index = sm->outside_fibs[0].fib_index;
3393       break;
3394     default:
3395       /* *INDENT-OFF* */
3396       vec_foreach (outside_fib, sm->outside_fibs)
3397         {
3398           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3399           if (FIB_NODE_INDEX_INVALID != fei)
3400             {
3401               if (fib_entry_get_resolving_interface (fei) != ~0)
3402                 {
3403                   key.fib_index = outside_fib->fib_index;
3404                   break;
3405                 }
3406             }
3407         }
3408       /* *INDENT-ON* */
3409       break;
3410     }
3411   key.addr.as_u32 = out_addr->as_u32;
3412   key.port = out_port;
3413   s->out2in = key;
3414   kv.value = s - tsm->sessions;
3415
3416   key.addr.as_u32 = in_addr->as_u32;
3417   key.port = in_port;
3418   key.fib_index = fib_index;
3419   s->in2out = key;
3420
3421   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3422               snat_proto_to_ip_proto (proto), fib_index, in_port,
3423               s->ext_host_nat_port);
3424   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3425     nat_log_warn ("in2out key add failed");
3426
3427   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3428               s->out2in.fib_index, out_port, eh_port);
3429   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3430     nat_log_warn ("out2in key add failed");
3431 }
3432
3433 void
3434 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3435                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3436                    u32 fib_index, u32 ti)
3437 {
3438   snat_main_t *sm = &snat_main;
3439   nat_ed_ses_key_t key;
3440   clib_bihash_kv_16_8_t kv, value;
3441   u32 thread_index;
3442   snat_session_t *s;
3443   snat_main_per_thread_data_t *tsm;
3444
3445   if (sm->num_workers > 1)
3446     thread_index =
3447       sm->first_worker_index +
3448       (sm->workers[(clib_net_to_host_u16 (out_port) -
3449                     1024) / sm->port_per_thread]);
3450   else
3451     thread_index = sm->num_workers;
3452   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3453
3454   key.l_addr.as_u32 = out_addr->as_u32;
3455   key.l_port = out_port;
3456   key.r_addr.as_u32 = eh_addr->as_u32;
3457   key.r_port = eh_port;
3458   key.proto = proto;
3459   key.fib_index = fib_index;
3460   kv.key[0] = key.as_u64[0];
3461   kv.key[1] = key.as_u64[1];
3462   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3463     return;
3464
3465   s = pool_elt_at_index (tsm->sessions, value.value);
3466   nat_free_session_data (sm, s, thread_index, 1);
3467   nat44_delete_session (sm, s, thread_index);
3468 }
3469
3470 void
3471 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3472                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3473                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3474                    u32 thread_index)
3475 {
3476   snat_main_t *sm = &snat_main;
3477   nat_ed_ses_key_t key;
3478   clib_bihash_kv_16_8_t kv, value;
3479   snat_session_t *s;
3480   snat_main_per_thread_data_t *tsm;
3481
3482   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3483
3484   key.l_addr.as_u32 = out_addr->as_u32;
3485   key.l_port = out_port;
3486   key.r_addr.as_u32 = eh_addr->as_u32;
3487   key.r_port = eh_port;
3488   key.proto = proto;
3489   key.fib_index = fib_index;
3490   kv.key[0] = key.as_u64[0];
3491   kv.key[1] = key.as_u64[1];
3492   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3493     return;
3494
3495   s = pool_elt_at_index (tsm->sessions, value.value);
3496   s->total_pkts = total_pkts;
3497   s->total_bytes = total_bytes;
3498 }
3499
3500 static clib_error_t *
3501 snat_config (vlib_main_t * vm, unformat_input_t * input)
3502 {
3503   snat_main_t *sm = &snat_main;
3504   nat66_main_t *nm = &nat66_main;
3505   u32 translation_buckets = 1024;
3506   u32 translation_memory_size = 128 << 20;
3507   u32 user_buckets = 128;
3508   u32 user_memory_size = 64 << 20;
3509   u32 max_translations_per_user = 100;
3510   u32 outside_vrf_id = 0;
3511   u32 outside_ip6_vrf_id = 0;
3512   u32 inside_vrf_id = 0;
3513   u32 static_mapping_buckets = 1024;
3514   u32 static_mapping_memory_size = 64 << 20;
3515   u32 nat64_bib_buckets = 1024;
3516   u32 nat64_bib_memory_size = 128 << 20;
3517   u32 nat64_st_buckets = 2048;
3518   u32 nat64_st_memory_size = 256 << 20;
3519   u8 static_mapping_only = 0;
3520   u8 static_mapping_connection_tracking = 0;
3521   snat_main_per_thread_data_t *tsm;
3522   dslite_main_t *dm = &dslite_main;
3523
3524   sm->deterministic = 0;
3525   sm->out2in_dpo = 0;
3526   sm->endpoint_dependent = 0;
3527
3528   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3529     {
3530       if (unformat
3531           (input, "translation hash buckets %d", &translation_buckets))
3532         ;
3533       else if (unformat (input, "translation hash memory %d",
3534                          &translation_memory_size));
3535       else if (unformat (input, "user hash buckets %d", &user_buckets))
3536         ;
3537       else if (unformat (input, "user hash memory %d", &user_memory_size))
3538         ;
3539       else if (unformat (input, "max translations per user %d",
3540                          &max_translations_per_user))
3541         ;
3542       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3543         ;
3544       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3545         ;
3546       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3547         ;
3548       else if (unformat (input, "static mapping only"))
3549         {
3550           static_mapping_only = 1;
3551           if (unformat (input, "connection tracking"))
3552             static_mapping_connection_tracking = 1;
3553         }
3554       else if (unformat (input, "deterministic"))
3555         sm->deterministic = 1;
3556       else if (unformat (input, "nat64 bib hash buckets %d",
3557                          &nat64_bib_buckets))
3558         ;
3559       else if (unformat (input, "nat64 bib hash memory %d",
3560                          &nat64_bib_memory_size))
3561         ;
3562       else
3563         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3564         ;
3565       else if (unformat (input, "nat64 st hash memory %d",
3566                          &nat64_st_memory_size))
3567         ;
3568       else if (unformat (input, "out2in dpo"))
3569         sm->out2in_dpo = 1;
3570       else if (unformat (input, "dslite ce"))
3571         dslite_set_ce (dm, 1);
3572       else if (unformat (input, "endpoint-dependent"))
3573         sm->endpoint_dependent = 1;
3574       else
3575         return clib_error_return (0, "unknown input '%U'",
3576                                   format_unformat_error, input);
3577     }
3578
3579   if (sm->deterministic && sm->endpoint_dependent)
3580     return clib_error_return (0,
3581                               "deterministic and endpoint-dependent modes are mutually exclusive");
3582
3583   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3584     return clib_error_return (0,
3585                               "static mapping only mode available only for simple nat");
3586
3587   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3588     return clib_error_return (0,
3589                               "out2in dpo mode available only for simple nat");
3590
3591   /* for show commands, etc. */
3592   sm->translation_buckets = translation_buckets;
3593   sm->translation_memory_size = translation_memory_size;
3594   /* do not exceed load factor 10 */
3595   sm->max_translations = 10 * translation_buckets;
3596   sm->user_buckets = user_buckets;
3597   sm->user_memory_size = user_memory_size;
3598   sm->max_translations_per_user = max_translations_per_user;
3599   sm->outside_vrf_id = outside_vrf_id;
3600   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3601                                                              outside_vrf_id,
3602                                                              FIB_SOURCE_PLUGIN_HI);
3603   nm->outside_vrf_id = outside_ip6_vrf_id;
3604   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3605                                                              outside_ip6_vrf_id,
3606                                                              FIB_SOURCE_PLUGIN_HI);
3607   sm->inside_vrf_id = inside_vrf_id;
3608   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3609                                                             inside_vrf_id,
3610                                                             FIB_SOURCE_PLUGIN_HI);
3611   sm->static_mapping_only = static_mapping_only;
3612   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3613
3614   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3615                   nat64_st_memory_size);
3616
3617   if (sm->deterministic)
3618     {
3619       sm->in2out_node_index = snat_det_in2out_node.index;
3620       sm->in2out_output_node_index = ~0;
3621       sm->out2in_node_index = snat_det_out2in_node.index;
3622       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3623       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3624     }
3625   else
3626     {
3627       if (sm->endpoint_dependent)
3628         {
3629           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3630           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3631           sm->in2out_node_index = nat44_ed_in2out_node.index;
3632           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3633           sm->out2in_node_index = nat44_ed_out2in_node.index;
3634           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3635           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3636           nat_affinity_init (vm);
3637           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
3638                        nat_ha_sref_ed_cb);
3639         }
3640       else
3641         {
3642           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3643           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3644           sm->in2out_node_index = snat_in2out_node.index;
3645           sm->in2out_output_node_index = snat_in2out_output_node.index;
3646           sm->out2in_node_index = snat_out2in_node.index;
3647           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3648           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3649           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
3650         }
3651       if (!static_mapping_only ||
3652           (static_mapping_only && static_mapping_connection_tracking))
3653         {
3654           /* *INDENT-OFF* */
3655           vec_foreach (tsm, sm->per_thread_data)
3656             {
3657               if (sm->endpoint_dependent)
3658                 {
3659                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3660                                          translation_buckets,
3661                                          translation_memory_size);
3662                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3663                                                       format_ed_session_kvp);
3664
3665                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3666                                          translation_buckets,
3667                                          translation_memory_size);
3668                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3669                                                       format_ed_session_kvp);
3670                 }
3671               else
3672                 {
3673                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3674                                         translation_buckets,
3675                                         translation_memory_size);
3676                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3677                                                      format_session_kvp);
3678
3679                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3680                                         translation_buckets,
3681                                         translation_memory_size);
3682                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3683                                                      format_session_kvp);
3684                 }
3685
3686               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3687                                     user_memory_size);
3688               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3689                                                  format_user_kvp);
3690             }
3691           /* *INDENT-ON* */
3692
3693         }
3694       else
3695         {
3696           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3697           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3698         }
3699       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3700                             "static_mapping_by_local", static_mapping_buckets,
3701                             static_mapping_memory_size);
3702       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3703                                          format_static_mapping_kvp);
3704
3705       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3706                             "static_mapping_by_external",
3707                             static_mapping_buckets,
3708                             static_mapping_memory_size);
3709       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3710                                          format_static_mapping_kvp);
3711     }
3712
3713   return 0;
3714 }
3715
3716 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3717
3718 static void
3719 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3720                                  uword opaque,
3721                                  u32 sw_if_index,
3722                                  ip4_address_t * address,
3723                                  u32 address_length,
3724                                  u32 if_address_index, u32 is_delete)
3725 {
3726   snat_main_t *sm = &snat_main;
3727   snat_static_map_resolve_t *rp;
3728   snat_static_mapping_t *m;
3729   snat_session_key_t m_key;
3730   clib_bihash_kv_8_8_t kv, value;
3731   int i, rv;
3732   ip4_address_t l_addr;
3733
3734   for (i = 0; i < vec_len (sm->to_resolve); i++)
3735     {
3736       rp = sm->to_resolve + i;
3737       if (rp->addr_only == 0)
3738         continue;
3739       if (rp->sw_if_index == sw_if_index)
3740         goto match;
3741     }
3742
3743   return;
3744
3745 match:
3746   m_key.addr.as_u32 = address->as_u32;
3747   m_key.port = rp->addr_only ? 0 : rp->e_port;
3748   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3749   m_key.fib_index = sm->outside_fib_index;
3750   kv.key = m_key.as_u64;
3751   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3752     m = 0;
3753   else
3754     m = pool_elt_at_index (sm->static_mappings, value.value);
3755
3756   if (!is_delete)
3757     {
3758       /* Don't trip over lease renewal, static config */
3759       if (m)
3760         return;
3761     }
3762   else
3763     {
3764       if (!m)
3765         return;
3766     }
3767
3768   /* Indetity mapping? */
3769   if (rp->l_addr.as_u32 == 0)
3770     l_addr.as_u32 = address[0].as_u32;
3771   else
3772     l_addr.as_u32 = rp->l_addr.as_u32;
3773   /* Add the static mapping */
3774   rv = snat_add_static_mapping (l_addr,
3775                                 address[0],
3776                                 rp->l_port,
3777                                 rp->e_port,
3778                                 rp->vrf_id,
3779                                 rp->addr_only, ~0 /* sw_if_index */ ,
3780                                 rp->proto, !is_delete, rp->twice_nat,
3781                                 rp->out2in_only, rp->tag, rp->identity_nat);
3782   if (rv)
3783     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3784 }
3785
3786 static void
3787 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3788                                        uword opaque,
3789                                        u32 sw_if_index,
3790                                        ip4_address_t * address,
3791                                        u32 address_length,
3792                                        u32 if_address_index, u32 is_delete)
3793 {
3794   snat_main_t *sm = &snat_main;
3795   snat_static_map_resolve_t *rp;
3796   ip4_address_t l_addr;
3797   int i, j;
3798   int rv;
3799   u8 twice_nat = 0;
3800   snat_address_t *addresses = sm->addresses;
3801
3802   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3803     {
3804       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3805         goto match;
3806     }
3807
3808   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3809     {
3810       twice_nat = 1;
3811       addresses = sm->twice_nat_addresses;
3812       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3813         goto match;
3814     }
3815
3816   return;
3817
3818 match:
3819   if (!is_delete)
3820     {
3821       /* Don't trip over lease renewal, static config */
3822       for (j = 0; j < vec_len (addresses); j++)
3823         if (addresses[j].addr.as_u32 == address->as_u32)
3824           return;
3825
3826       (void) snat_add_address (sm, address, ~0, twice_nat);
3827       /* Scan static map resolution vector */
3828       for (j = 0; j < vec_len (sm->to_resolve); j++)
3829         {
3830           rp = sm->to_resolve + j;
3831           if (rp->addr_only)
3832             continue;
3833           /* On this interface? */
3834           if (rp->sw_if_index == sw_if_index)
3835             {
3836               /* Indetity mapping? */
3837               if (rp->l_addr.as_u32 == 0)
3838                 l_addr.as_u32 = address[0].as_u32;
3839               else
3840                 l_addr.as_u32 = rp->l_addr.as_u32;
3841               /* Add the static mapping */
3842               rv = snat_add_static_mapping (l_addr,
3843                                             address[0],
3844                                             rp->l_port,
3845                                             rp->e_port,
3846                                             rp->vrf_id,
3847                                             rp->addr_only,
3848                                             ~0 /* sw_if_index */ ,
3849                                             rp->proto,
3850                                             rp->is_add, rp->twice_nat,
3851                                             rp->out2in_only, rp->tag,
3852                                             rp->identity_nat);
3853               if (rv)
3854                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3855             }
3856         }
3857       return;
3858     }
3859   else
3860     {
3861       (void) snat_del_address (sm, address[0], 1, twice_nat);
3862       return;
3863     }
3864 }
3865
3866
3867 int
3868 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3869                             u8 twice_nat)
3870 {
3871   ip4_main_t *ip4_main = sm->ip4_main;
3872   ip4_address_t *first_int_addr;
3873   snat_static_map_resolve_t *rp;
3874   u32 *indices_to_delete = 0;
3875   int i, j;
3876   u32 *auto_add_sw_if_indices =
3877     twice_nat ? sm->
3878     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3879
3880   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3881     );
3882
3883   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3884     {
3885       if (auto_add_sw_if_indices[i] == sw_if_index)
3886         {
3887           if (is_del)
3888             {
3889               /* if have address remove it */
3890               if (first_int_addr)
3891                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3892               else
3893                 {
3894                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3895                     {
3896                       rp = sm->to_resolve + j;
3897                       if (rp->sw_if_index == sw_if_index)
3898                         vec_add1 (indices_to_delete, j);
3899                     }
3900                   if (vec_len (indices_to_delete))
3901                     {
3902                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3903                         vec_del1 (sm->to_resolve, j);
3904                       vec_free (indices_to_delete);
3905                     }
3906                 }
3907               if (twice_nat)
3908                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3909               else
3910                 vec_del1 (sm->auto_add_sw_if_indices, i);
3911             }
3912           else
3913             return VNET_API_ERROR_VALUE_EXIST;
3914
3915           return 0;
3916         }
3917     }
3918
3919   if (is_del)
3920     return VNET_API_ERROR_NO_SUCH_ENTRY;
3921
3922   /* add to the auto-address list */
3923   if (twice_nat)
3924     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3925   else
3926     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3927
3928   /* If the address is already bound - or static - add it now */
3929   if (first_int_addr)
3930     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3931
3932   return 0;
3933 }
3934
3935 int
3936 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3937                    snat_protocol_t proto, u32 vrf_id, int is_in)
3938 {
3939   snat_main_per_thread_data_t *tsm;
3940   clib_bihash_kv_8_8_t kv, value;
3941   ip4_header_t ip;
3942   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3943   snat_session_key_t key;
3944   snat_session_t *s;
3945   clib_bihash_8_8_t *t;
3946
3947   if (sm->endpoint_dependent)
3948     return VNET_API_ERROR_UNSUPPORTED;
3949
3950   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3951   if (sm->num_workers > 1)
3952     tsm =
3953       vec_elt_at_index (sm->per_thread_data,
3954                         sm->worker_in2out_cb (&ip, fib_index));
3955   else
3956     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3957
3958   key.addr.as_u32 = addr->as_u32;
3959   key.port = clib_host_to_net_u16 (port);
3960   key.protocol = proto;
3961   key.fib_index = fib_index;
3962   kv.key = key.as_u64;
3963   t = is_in ? &tsm->in2out : &tsm->out2in;
3964   if (!clib_bihash_search_8_8 (t, &kv, &value))
3965     {
3966       if (pool_is_free_index (tsm->sessions, value.value))
3967         return VNET_API_ERROR_UNSPECIFIED;
3968
3969       s = pool_elt_at_index (tsm->sessions, value.value);
3970       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3971       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3972       return 0;
3973     }
3974
3975   return VNET_API_ERROR_NO_SUCH_ENTRY;
3976 }
3977
3978 int
3979 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3980                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3981                       u32 vrf_id, int is_in)
3982 {
3983   ip4_header_t ip;
3984   clib_bihash_16_8_t *t;
3985   nat_ed_ses_key_t key;
3986   clib_bihash_kv_16_8_t kv, value;
3987   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3988   snat_session_t *s;
3989   snat_main_per_thread_data_t *tsm;
3990
3991   if (!sm->endpoint_dependent)
3992     return VNET_API_ERROR_FEATURE_DISABLED;
3993
3994   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3995   if (sm->num_workers > 1)
3996     tsm =
3997       vec_elt_at_index (sm->per_thread_data,
3998                         sm->worker_in2out_cb (&ip, fib_index));
3999   else
4000     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4001
4002   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4003   key.l_addr.as_u32 = addr->as_u32;
4004   key.r_addr.as_u32 = eh_addr->as_u32;
4005   key.l_port = clib_host_to_net_u16 (port);
4006   key.r_port = clib_host_to_net_u16 (eh_port);
4007   key.proto = proto;
4008   key.fib_index = fib_index;
4009   kv.key[0] = key.as_u64[0];
4010   kv.key[1] = key.as_u64[1];
4011   if (clib_bihash_search_16_8 (t, &kv, &value))
4012     return VNET_API_ERROR_NO_SUCH_ENTRY;
4013
4014   if (pool_is_free_index (tsm->sessions, value.value))
4015     return VNET_API_ERROR_UNSPECIFIED;
4016   s = pool_elt_at_index (tsm->sessions, value.value);
4017   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4018   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4019   return 0;
4020 }
4021
4022 void
4023 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4024 {
4025   snat_main_t *sm = &snat_main;
4026
4027   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4028   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4029   sm->psid = psid;
4030   sm->psid_offset = psid_offset;
4031   sm->psid_length = psid_length;
4032 }
4033
4034 void
4035 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4036 {
4037   snat_main_t *sm = &snat_main;
4038
4039   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4040   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4041   sm->start_port = start_port;
4042   sm->end_port = end_port;
4043 }
4044
4045 void
4046 nat_set_alloc_addr_and_port_default (void)
4047 {
4048   snat_main_t *sm = &snat_main;
4049
4050   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4051   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4052 }
4053
4054 /*
4055  * fd.io coding-style-patch-verification: ON
4056  *
4057  * Local Variables:
4058  * eval: (c-set-style "gnu")
4059  * End:
4060  */