src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <vnet/fib/fib_table.h>
  35 #include <vnet/fib/ip4_fib.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 /* *INDENT-OFF* */
  42
  43 /* Hook up input features */
  44 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  45   .arc_name = "ip4-unicast",
  46   .node_name = "nat44-in2out",
  47   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  48 };
  49 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  50   .arc_name = "ip4-unicast",
  51   .node_name = "nat44-out2in",
  52   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  53                                "ip4-dhcp-client-detect"),
  54 };
  55 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  56   .arc_name = "ip4-unicast",
  57   .node_name = "nat44-classify",
  58   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  59 };
  60 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  61   .arc_name = "ip4-unicast",
  62   .node_name = "nat44-det-in2out",
  63   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  64 };
  65 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  66   .arc_name = "ip4-unicast",
  67   .node_name = "nat44-det-out2in",
  68   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  69                                "ip4-dhcp-client-detect"),
  70 };
  71 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  72   .arc_name = "ip4-unicast",
  73   .node_name = "nat44-det-classify",
  74   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  75 };
  76 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  77   .arc_name = "ip4-unicast",
  78   .node_name = "nat44-ed-in2out",
  79   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-ed-out2in",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  85                                "ip4-dhcp-client-detect"),
  86 };
  87 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  88   .arc_name = "ip4-unicast",
  89   .node_name = "nat44-ed-classify",
  90   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  91 };
  92 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-in2out-worker-handoff",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  96 };
  97 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-out2in-worker-handoff",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 101                                "ip4-dhcp-client-detect"),
 102 };
 103 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-handoff-classify",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 107 };
 108 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-in2out-fast",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 112 };
 113 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-out2in-fast",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 117                                "ip4-dhcp-client-detect"),
 118 };
 119 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-hairpin-dst",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 123 };
 124 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-ed-hairpin-dst",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 128 };
 129
 130 /* Hook up output features */
 131 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 132   .arc_name = "ip4-output",
 133   .node_name = "nat44-in2out-output",
 134   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 137   .arc_name = "ip4-output",
 138   .node_name = "nat44-in2out-output-worker-handoff",
 139   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 140 };
 141 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 142   .arc_name = "ip4-output",
 143   .node_name = "nat44-hairpin-src",
 144   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 145 };
 146 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-ed-in2out-output",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 150 };
 151 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-ed-hairpin-src",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 155 };
 156
 157 /* Hook up ip4-local features */
 158 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 159 {
 160   .arc_name = "ip4-local",
 161   .node_name = "nat44-hairpinning",
 162   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 163 };
 164 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 165 {
 166   .arc_name = "ip4-local",
 167   .node_name = "nat44-ed-hairpinning",
 168   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 169 };
 170
 171
 172 VLIB_PLUGIN_REGISTER () = {
 173     .version = VPP_BUILD_VER,
 174     .description = "Network Address Translation (NAT)",
 175 };
 176 /* *INDENT-ON* */
 177
 178 void
 179 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 180                        u8 is_ha)
 181 {
 182   snat_session_key_t key;
 183   clib_bihash_kv_8_8_t kv;
 184   nat_ed_ses_key_t ed_key;
 185   clib_bihash_kv_16_8_t ed_kv;
 186   snat_main_per_thread_data_t *tsm =
 187     vec_elt_at_index (sm->per_thread_data, thread_index);
 188
 189   if (is_fwd_bypass_session (s))
 190     {
 191       if (snat_is_unk_proto_session (s))
 192         {
 193           ed_key.proto = s->in2out.port;
 194           ed_key.r_port = 0;
 195           ed_key.l_port = 0;
 196         }
 197       else
 198         {
 199           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 200           ed_key.l_port = s->in2out.port;
 201           ed_key.r_port = s->ext_host_port;
 202         }
 203       ed_key.l_addr = s->in2out.addr;
 204       ed_key.r_addr = s->ext_host_addr;
 205       ed_key.fib_index = 0;
 206       ed_kv.key[0] = ed_key.as_u64[0];
 207       ed_kv.key[1] = ed_key.as_u64[1];
 208       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 209         nat_elog_warn ("in2out_ed key del failed");
 210       return;
 211     }
 212
 213   /* session lookup tables */
 214   if (is_ed_session (s))
 215     {
 216       if (is_affinity_sessions (s))
 217         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 218                              s->in2out.protocol, s->out2in.port);
 219       ed_key.l_addr = s->out2in.addr;
 220       ed_key.r_addr = s->ext_host_addr;
 221       ed_key.fib_index = s->out2in.fib_index;
 222       if (snat_is_unk_proto_session (s))
 223         {
 224           ed_key.proto = s->in2out.port;
 225           ed_key.r_port = 0;
 226           ed_key.l_port = 0;
 227         }
 228       else
 229         {
 230           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 231           ed_key.l_port = s->out2in.port;
 232           ed_key.r_port = s->ext_host_port;
 233         }
 234       ed_kv.key[0] = ed_key.as_u64[0];
 235       ed_kv.key[1] = ed_key.as_u64[1];
 236       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 237         nat_elog_warn ("out2in_ed key del failed");
 238       ed_key.l_addr = s->in2out.addr;
 239       ed_key.fib_index = s->in2out.fib_index;
 240       if (!snat_is_unk_proto_session (s))
 241         ed_key.l_port = s->in2out.port;
 242       if (is_twice_nat_session (s))
 243         {
 244           ed_key.r_addr = s->ext_host_nat_addr;
 245           ed_key.r_port = s->ext_host_nat_port;
 246         }
 247       ed_kv.key[0] = ed_key.as_u64[0];
 248       ed_kv.key[1] = ed_key.as_u64[1];
 249       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 250         nat_elog_warn ("in2out_ed key del failed");
 251
 252       if (!is_ha)
 253         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 254                                &s->in2out.addr, s->in2out.port,
 255                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 256                                &s->out2in.addr, s->out2in.port,
 257                                &s->ext_host_addr, s->ext_host_port,
 258                                s->in2out.protocol, is_twice_nat_session (s));
 259     }
 260   else
 261     {
 262       kv.key = s->in2out.as_u64;
 263       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 264         nat_elog_warn ("in2out key del failed");
 265       kv.key = s->out2in.as_u64;
 266       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 267         nat_elog_warn ("out2in key del failed");
 268
 269       if (!is_ha)
 270         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 271                                  &s->in2out.addr, s->in2out.port,
 272                                  &s->out2in.addr, s->out2in.port,
 273                                  s->in2out.protocol);
 274     }
 275
 276   if (snat_is_unk_proto_session (s))
 277     return;
 278
 279   if (!is_ha)
 280     {
 281       /* log NAT event */
 282       snat_ipfix_logging_nat44_ses_delete (thread_index,
 283                                            s->in2out.addr.as_u32,
 284                                            s->out2in.addr.as_u32,
 285                                            s->in2out.protocol,
 286                                            s->in2out.port,
 287                                            s->out2in.port,
 288                                            s->in2out.fib_index);
 289
 290       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 291                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 292                    thread_index);
 293     }
 294
 295   /* Twice NAT address and port for external host */
 296   if (is_twice_nat_session (s))
 297     {
 298       key.protocol = s->in2out.protocol;
 299       key.port = s->ext_host_nat_port;
 300       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 301       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 302                                           thread_index, &key);
 303     }
 304
 305   if (snat_is_session_static (s))
 306     return;
 307
 308   snat_free_outside_address_and_port (sm->addresses, thread_index,
 309                                       &s->out2in);
 310 }
 311
 312 snat_user_t *
 313 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 314                         u32 thread_index)
 315 {
 316   snat_user_t *u = 0;
 317   snat_user_key_t user_key;
 318   clib_bihash_kv_8_8_t kv, value;
 319   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 320   dlist_elt_t *per_user_list_head_elt;
 321
 322   user_key.addr.as_u32 = addr->as_u32;
 323   user_key.fib_index = fib_index;
 324   kv.key = user_key.as_u64;
 325
 326   /* Ever heard of the "user" = src ip4 address before? */
 327   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 328     {
 329       /* no, make a new one */
 330       pool_get (tsm->users, u);
 331       clib_memset (u, 0, sizeof (*u));
 332       u->addr.as_u32 = addr->as_u32;
 333       u->fib_index = fib_index;
 334
 335       pool_get (tsm->list_pool, per_user_list_head_elt);
 336
 337       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 338         tsm->list_pool;
 339
 340       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 341
 342       kv.value = u - tsm->users;
 343
 344       /* add user */
 345       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 346         nat_elog_warn ("user_hash keay add failed");
 347
 348       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 349                                pool_elts (tsm->users));
 350     }
 351   else
 352     {
 353       u = pool_elt_at_index (tsm->users, value.value);
 354     }
 355
 356   return u;
 357 }
 358
 359 snat_session_t *
 360 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 361                               u32 thread_index, f64 now)
 362 {
 363   snat_session_t *s;
 364   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 365   u32 oldest_per_user_translation_list_index, session_index;
 366   dlist_elt_t *oldest_per_user_translation_list_elt;
 367   dlist_elt_t *per_user_translation_list_elt;
 368
 369   /* Over quota? Recycle the least recently used translation */
 370   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 371     {
 372       oldest_per_user_translation_list_index =
 373         clib_dlist_remove_head (tsm->list_pool,
 374                                 u->sessions_per_user_list_head_index);
 375
 376       ASSERT (oldest_per_user_translation_list_index != ~0);
 377
 378       /* Add it back to the end of the LRU list */
 379       clib_dlist_addtail (tsm->list_pool,
 380                           u->sessions_per_user_list_head_index,
 381                           oldest_per_user_translation_list_index);
 382       /* Get the list element */
 383       oldest_per_user_translation_list_elt =
 384         pool_elt_at_index (tsm->list_pool,
 385                            oldest_per_user_translation_list_index);
 386
 387       /* Get the session index from the list element */
 388       session_index = oldest_per_user_translation_list_elt->value;
 389
 390       /* Get the session */
 391       s = pool_elt_at_index (tsm->sessions, session_index);
 392       nat_free_session_data (sm, s, thread_index, 0);
 393       if (snat_is_session_static (s))
 394         u->nstaticsessions--;
 395       else
 396         u->nsessions--;
 397       s->flags = 0;
 398       s->total_bytes = 0;
 399       s->total_pkts = 0;
 400       s->state = 0;
 401       s->ext_host_addr.as_u32 = 0;
 402       s->ext_host_port = 0;
 403       s->ext_host_nat_addr.as_u32 = 0;
 404       s->ext_host_nat_port = 0;
 405     }
 406   else
 407     {
 408       pool_get (tsm->sessions, s);
 409       clib_memset (s, 0, sizeof (*s));
 410
 411       /* Create list elts */
 412       pool_get (tsm->list_pool, per_user_translation_list_elt);
 413       clib_dlist_init (tsm->list_pool,
 414                        per_user_translation_list_elt - tsm->list_pool);
 415
 416       per_user_translation_list_elt->value = s - tsm->sessions;
 417       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 418       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 419
 420       clib_dlist_addtail (tsm->list_pool,
 421                           s->per_user_list_head_index,
 422                           per_user_translation_list_elt - tsm->list_pool);
 423
 424       s->user_index = u - tsm->users;
 425       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 426                                pool_elts (tsm->sessions));
 427     }
 428
 429   s->ha_last_refreshed = now;
 430
 431   return s;
 432 }
 433
 434 snat_session_t *
 435 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 436                       f64 now)
 437 {
 438   snat_session_t *s;
 439   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 440   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 441   u32 oldest_index;
 442   u64 sess_timeout_time;
 443
 444   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 445     goto alloc_new;
 446
 447   oldest_index =
 448     clib_dlist_remove_head (tsm->list_pool,
 449                             u->sessions_per_user_list_head_index);
 450   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 451   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 452   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 453   if (now >= sess_timeout_time)
 454     {
 455       clib_dlist_addtail (tsm->list_pool,
 456                           u->sessions_per_user_list_head_index, oldest_index);
 457       nat_free_session_data (sm, s, thread_index, 0);
 458       if (snat_is_session_static (s))
 459         u->nstaticsessions--;
 460       else
 461         u->nsessions--;
 462       s->flags = 0;
 463       s->total_bytes = 0;
 464       s->total_pkts = 0;
 465       s->state = 0;
 466       s->ext_host_addr.as_u32 = 0;
 467       s->ext_host_port = 0;
 468       s->ext_host_nat_addr.as_u32 = 0;
 469       s->ext_host_nat_port = 0;
 470     }
 471   else
 472     {
 473       clib_dlist_addhead (tsm->list_pool,
 474                           u->sessions_per_user_list_head_index, oldest_index);
 475       if ((u->nsessions + u->nstaticsessions) >=
 476           sm->max_translations_per_user)
 477         {
 478           nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 479                          clib_net_to_host_u32 (u->addr.as_u32));
 480           snat_ipfix_logging_max_entries_per_user
 481             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 482           return 0;
 483         }
 484       else
 485         {
 486         alloc_new:
 487           pool_get (tsm->sessions, s);
 488           clib_memset (s, 0, sizeof (*s));
 489
 490           /* Create list elts */
 491           pool_get (tsm->list_pool, per_user_translation_list_elt);
 492           clib_dlist_init (tsm->list_pool,
 493                            per_user_translation_list_elt - tsm->list_pool);
 494
 495           per_user_translation_list_elt->value = s - tsm->sessions;
 496           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 497           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 498
 499           clib_dlist_addtail (tsm->list_pool,
 500                               s->per_user_list_head_index,
 501                               per_user_translation_list_elt - tsm->list_pool);
 502         }
 503
 504       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 505                                pool_elts (tsm->sessions));
 506     }
 507
 508   s->ha_last_refreshed = now;
 509
 510   return s;
 511 }
 512
 513 void
 514 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 515                           int is_add)
 516 {
 517   fib_prefix_t prefix = {
 518     .fp_len = p_len,
 519     .fp_proto = FIB_PROTOCOL_IP4,
 520     .fp_addr = {
 521                 .ip4.as_u32 = addr->as_u32,
 522                 },
 523   };
 524   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 525
 526   if (is_add)
 527     fib_table_entry_update_one_path (fib_index,
 528                                      &prefix,
 529                                      FIB_SOURCE_PLUGIN_LOW,
 530                                      (FIB_ENTRY_FLAG_CONNECTED |
 531                                       FIB_ENTRY_FLAG_LOCAL |
 532                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 533                                      DPO_PROTO_IP4,
 534                                      NULL,
 535                                      sw_if_index,
 536                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 537   else
 538     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 539 }
 540
 541 int
 542 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 543                   u8 twice_nat)
 544 {
 545   snat_address_t *ap;
 546   snat_interface_t *i;
 547   vlib_thread_main_t *tm = vlib_get_thread_main ();
 548
 549   if (twice_nat && !sm->endpoint_dependent)
 550     return VNET_API_ERROR_FEATURE_DISABLED;
 551
 552   /* Check if address already exists */
 553   /* *INDENT-OFF* */
 554   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 555     {
 556       if (ap->addr.as_u32 == addr->as_u32)
 557         return VNET_API_ERROR_VALUE_EXIST;
 558     }
 559   /* *INDENT-ON* */
 560
 561   if (twice_nat)
 562     vec_add2 (sm->twice_nat_addresses, ap, 1);
 563   else
 564     vec_add2 (sm->addresses, ap, 1);
 565
 566   ap->addr = *addr;
 567   if (vrf_id != ~0)
 568     ap->fib_index =
 569       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 570                                          FIB_SOURCE_PLUGIN_LOW);
 571   else
 572     ap->fib_index = ~0;
 573 #define _(N, i, n, s) \
 574   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 575   ap->busy_##n##_ports = 0; \
 576   ap->busy_##n##_ports_per_thread = 0;\
 577   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 578   foreach_snat_protocol
 579 #undef _
 580     if (twice_nat)
 581     return 0;
 582
 583   /* Add external address to FIB */
 584   /* *INDENT-OFF* */
 585   pool_foreach (i, sm->interfaces,
 586   ({
 587     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 588       continue;
 589
 590     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 591     break;
 592   }));
 593   pool_foreach (i, sm->output_feature_interfaces,
 594   ({
 595     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 596       continue;
 597
 598     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 599     break;
 600   }));
 601   /* *INDENT-ON* */
 602
 603   return 0;
 604 }
 605
 606 static int
 607 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 608 {
 609   snat_static_mapping_t *m;
 610   /* *INDENT-OFF* */
 611   pool_foreach (m, sm->static_mappings,
 612   ({
 613       if (is_addr_only_static_mapping (m) ||
 614           is_out2in_only_static_mapping (m) ||
 615           is_identity_static_mapping (m))
 616         continue;
 617       if (m->external_addr.as_u32 == addr.as_u32)
 618         return 1;
 619   }));
 620   /* *INDENT-ON* */
 621
 622   return 0;
 623 }
 624
 625 void
 626 increment_v4_address (ip4_address_t * a)
 627 {
 628   u32 v;
 629
 630   v = clib_net_to_host_u32 (a->as_u32) + 1;
 631   a->as_u32 = clib_host_to_net_u32 (v);
 632 }
 633
 634 static void
 635 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 636                                        ip4_address_t l_addr,
 637                                        u16 l_port,
 638                                        u32 sw_if_index,
 639                                        u16 e_port,
 640                                        u32 vrf_id,
 641                                        snat_protocol_t proto,
 642                                        int addr_only, int is_add, u8 * tag,
 643                                        int twice_nat, int out2in_only,
 644                                        int identity_nat)
 645 {
 646   snat_static_map_resolve_t *rp;
 647
 648   vec_add2 (sm->to_resolve, rp, 1);
 649   rp->l_addr.as_u32 = l_addr.as_u32;
 650   rp->l_port = l_port;
 651   rp->sw_if_index = sw_if_index;
 652   rp->e_port = e_port;
 653   rp->vrf_id = vrf_id;
 654   rp->proto = proto;
 655   rp->addr_only = addr_only;
 656   rp->is_add = is_add;
 657   rp->twice_nat = twice_nat;
 658   rp->out2in_only = out2in_only;
 659   rp->identity_nat = identity_nat;
 660   rp->tag = vec_dup (tag);
 661 }
 662
 663 static u32
 664 get_thread_idx_by_port (u16 e_port)
 665 {
 666   snat_main_t *sm = &snat_main;
 667   u32 thread_idx = sm->num_workers;
 668   if (sm->num_workers > 1)
 669     {
 670       thread_idx =
 671         sm->first_worker_index +
 672         sm->workers[(e_port - 1024) / sm->port_per_thread];
 673     }
 674   return thread_idx;
 675 }
 676
 677 int
 678 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 679                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 680                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 681                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 682                          u8 identity_nat)
 683 {
 684   snat_main_t *sm = &snat_main;
 685   snat_static_mapping_t *m;
 686   snat_session_key_t m_key;
 687   clib_bihash_kv_8_8_t kv, value;
 688   snat_address_t *a = 0;
 689   u32 fib_index = ~0;
 690   snat_interface_t *interface;
 691   int i;
 692   snat_main_per_thread_data_t *tsm;
 693   snat_user_key_t u_key;
 694   snat_user_t *u;
 695   dlist_elt_t *head, *elt;
 696   u32 elt_index, head_index;
 697   u32 ses_index;
 698   u64 user_index;
 699   snat_session_t *s;
 700   snat_static_map_resolve_t *rp, *rp_match = 0;
 701   nat44_lb_addr_port_t *local;
 702   u32 find = ~0;
 703
 704   if (!sm->endpoint_dependent)
 705     {
 706       if (twice_nat || out2in_only)
 707         return VNET_API_ERROR_FEATURE_DISABLED;
 708     }
 709
 710   /* If the external address is a specific interface address */
 711   if (sw_if_index != ~0)
 712     {
 713       ip4_address_t *first_int_addr;
 714
 715       for (i = 0; i < vec_len (sm->to_resolve); i++)
 716         {
 717           rp = sm->to_resolve + i;
 718           if (rp->sw_if_index != sw_if_index ||
 719               rp->l_addr.as_u32 != l_addr.as_u32 ||
 720               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 721             continue;
 722
 723           if (!addr_only)
 724             {
 725               if ((rp->l_port != l_port && rp->e_port != e_port)
 726                   || rp->proto != proto)
 727                 continue;
 728             }
 729
 730           rp_match = rp;
 731           break;
 732         }
 733
 734       /* Might be already set... */
 735       first_int_addr = ip4_interface_first_address
 736         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 737
 738       if (is_add)
 739         {
 740           if (rp_match)
 741             return VNET_API_ERROR_VALUE_EXIST;
 742
 743           snat_add_static_mapping_when_resolved
 744             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 745              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 746
 747           /* DHCP resolution required? */
 748           if (first_int_addr == 0)
 749             {
 750               return 0;
 751             }
 752           else
 753             {
 754               e_addr.as_u32 = first_int_addr->as_u32;
 755               /* Identity mapping? */
 756               if (l_addr.as_u32 == 0)
 757                 l_addr.as_u32 = e_addr.as_u32;
 758             }
 759         }
 760       else
 761         {
 762           if (!rp_match)
 763             return VNET_API_ERROR_NO_SUCH_ENTRY;
 764
 765           vec_del1 (sm->to_resolve, i);
 766
 767           if (first_int_addr)
 768             {
 769               e_addr.as_u32 = first_int_addr->as_u32;
 770               /* Identity mapping? */
 771               if (l_addr.as_u32 == 0)
 772                 l_addr.as_u32 = e_addr.as_u32;
 773             }
 774           else
 775             return 0;
 776         }
 777     }
 778
 779   m_key.addr = e_addr;
 780   m_key.port = addr_only ? 0 : e_port;
 781   m_key.protocol = addr_only ? 0 : proto;
 782   m_key.fib_index = 0;
 783   kv.key = m_key.as_u64;
 784   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 785     m = 0;
 786   else
 787     m = pool_elt_at_index (sm->static_mappings, value.value);
 788
 789   if (is_add)
 790     {
 791       if (m)
 792         {
 793           if (is_identity_static_mapping (m))
 794             {
 795               /* *INDENT-OFF* */
 796               pool_foreach (local, m->locals,
 797               ({
 798                 if (local->vrf_id == vrf_id)
 799                   return VNET_API_ERROR_VALUE_EXIST;
 800               }));
 801               /* *INDENT-ON* */
 802               pool_get (m->locals, local);
 803               local->vrf_id = vrf_id;
 804               local->fib_index =
 805                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 806                                                    FIB_SOURCE_PLUGIN_LOW);
 807               m_key.addr = m->local_addr;
 808               m_key.port = m->local_port;
 809               m_key.protocol = m->proto;
 810               m_key.fib_index = local->fib_index;
 811               kv.key = m_key.as_u64;
 812               kv.value = m - sm->static_mappings;
 813               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 814               return 0;
 815             }
 816           else
 817             return VNET_API_ERROR_VALUE_EXIST;
 818         }
 819
 820       if (twice_nat && addr_only)
 821         return VNET_API_ERROR_UNSUPPORTED;
 822
 823       /* Convert VRF id to FIB index */
 824       if (vrf_id != ~0)
 825         fib_index =
 826           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 827                                              FIB_SOURCE_PLUGIN_LOW);
 828       /* If not specified use inside VRF id from SNAT plugin startup config */
 829       else
 830         {
 831           fib_index = sm->inside_fib_index;
 832           vrf_id = sm->inside_vrf_id;
 833           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
 834         }
 835
 836       if (!(out2in_only || identity_nat))
 837         {
 838           m_key.addr = l_addr;
 839           m_key.port = addr_only ? 0 : l_port;
 840           m_key.protocol = addr_only ? 0 : proto;
 841           m_key.fib_index = fib_index;
 842           kv.key = m_key.as_u64;
 843           if (!clib_bihash_search_8_8
 844               (&sm->static_mapping_by_local, &kv, &value))
 845             return VNET_API_ERROR_VALUE_EXIST;
 846         }
 847
 848       /* Find external address in allocated addresses and reserve port for
 849          address and port pair mapping when dynamic translations enabled */
 850       if (!(addr_only || sm->static_mapping_only || out2in_only))
 851         {
 852           for (i = 0; i < vec_len (sm->addresses); i++)
 853             {
 854               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 855                 {
 856                   a = sm->addresses + i;
 857                   /* External port must be unused */
 858                   switch (proto)
 859                     {
 860 #define _(N, j, n, s) \
 861                     case SNAT_PROTOCOL_##N: \
 862                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 863                         return VNET_API_ERROR_INVALID_VALUE; \
 864                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 865                       if (e_port > 1024) \
 866                         { \
 867                           a->busy_##n##_ports++; \
 868                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 869                         } \
 870                       break;
 871                       foreach_snat_protocol
 872 #undef _
 873                     default:
 874                       nat_elog_info ("unknown protocol");
 875                       return VNET_API_ERROR_INVALID_VALUE_2;
 876                     }
 877                   break;
 878                 }
 879             }
 880           /* External address must be allocated */
 881           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 882             {
 883               if (sw_if_index != ~0)
 884                 {
 885                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 886                     {
 887                       rp = sm->to_resolve + i;
 888                       if (rp->addr_only)
 889                         continue;
 890                       if (rp->sw_if_index != sw_if_index &&
 891                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 892                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 893                           rp->e_port != e_port && rp->proto != proto)
 894                         continue;
 895
 896                       vec_del1 (sm->to_resolve, i);
 897                       break;
 898                     }
 899                 }
 900               return VNET_API_ERROR_NO_SUCH_ENTRY;
 901             }
 902         }
 903
 904       pool_get (sm->static_mappings, m);
 905       clib_memset (m, 0, sizeof (*m));
 906       m->tag = vec_dup (tag);
 907       m->local_addr = l_addr;
 908       m->external_addr = e_addr;
 909       m->twice_nat = twice_nat;
 910       if (out2in_only)
 911         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 912       if (addr_only)
 913         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 914       if (identity_nat)
 915         {
 916           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 917           pool_get (m->locals, local);
 918           local->vrf_id = vrf_id;
 919           local->fib_index = fib_index;
 920         }
 921       else
 922         {
 923           m->vrf_id = vrf_id;
 924           m->fib_index = fib_index;
 925         }
 926       if (!addr_only)
 927         {
 928           m->local_port = l_port;
 929           m->external_port = e_port;
 930           m->proto = proto;
 931         }
 932
 933       if (sm->num_workers > 1)
 934         {
 935           ip4_header_t ip = {
 936             .src_address = m->local_addr,
 937           };
 938           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 939           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 940         }
 941       else
 942         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 943
 944       m_key.addr = m->local_addr;
 945       m_key.port = m->local_port;
 946       m_key.protocol = m->proto;
 947       m_key.fib_index = fib_index;
 948       kv.key = m_key.as_u64;
 949       kv.value = m - sm->static_mappings;
 950       if (!out2in_only)
 951         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 952
 953       m_key.addr = m->external_addr;
 954       m_key.port = m->external_port;
 955       m_key.fib_index = 0;
 956       kv.key = m_key.as_u64;
 957       kv.value = m - sm->static_mappings;
 958       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 959
 960       /* Delete dynamic sessions matching local address (+ local port) */
 961       if (!(sm->static_mapping_only))
 962         {
 963           u_key.addr = m->local_addr;
 964           u_key.fib_index = m->fib_index;
 965           kv.key = u_key.as_u64;
 966           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 967             {
 968               user_index = value.value;
 969               u = pool_elt_at_index (tsm->users, user_index);
 970               if (u->nsessions)
 971                 {
 972                   head_index = u->sessions_per_user_list_head_index;
 973                   head = pool_elt_at_index (tsm->list_pool, head_index);
 974                   elt_index = head->next;
 975                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 976                   ses_index = elt->value;
 977                   while (ses_index != ~0)
 978                     {
 979                       s = pool_elt_at_index (tsm->sessions, ses_index);
 980                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 981                       ses_index = elt->value;
 982
 983                       if (snat_is_session_static (s))
 984                         continue;
 985
 986                       if (!addr_only
 987                           && (clib_net_to_host_u16 (s->in2out.port) !=
 988                               m->local_port))
 989                         continue;
 990
 991                       nat_free_session_data (sm, s,
 992                                              tsm - sm->per_thread_data, 0);
 993                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 994
 995                       if (!addr_only && !sm->endpoint_dependent)
 996                         break;
 997                     }
 998                 }
 999             }
1000         }
1001     }
1002   else
1003     {
1004       if (!m)
1005         {
1006           if (sw_if_index != ~0)
1007             return 0;
1008           else
1009             return VNET_API_ERROR_NO_SUCH_ENTRY;
1010         }
1011
1012       if (identity_nat)
1013         {
1014           if (vrf_id == ~0)
1015             vrf_id = sm->inside_vrf_id;
1016
1017           /* *INDENT-OFF* */
1018           pool_foreach (local, m->locals,
1019           ({
1020             if (local->vrf_id == vrf_id)
1021               find = local - m->locals;
1022           }));
1023           /* *INDENT-ON* */
1024           if (find == ~0)
1025             return VNET_API_ERROR_NO_SUCH_ENTRY;
1026
1027           local = pool_elt_at_index (m->locals, find);
1028           fib_index = local->fib_index;
1029           pool_put (m->locals, local);
1030         }
1031       else
1032         fib_index = m->fib_index;
1033
1034       /* Free external address port */
1035       if (!(addr_only || sm->static_mapping_only || out2in_only))
1036         {
1037           for (i = 0; i < vec_len (sm->addresses); i++)
1038             {
1039               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1040                 {
1041                   a = sm->addresses + i;
1042                   switch (proto)
1043                     {
1044 #define _(N, j, n, s) \
1045                     case SNAT_PROTOCOL_##N: \
1046                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1047                       if (e_port > 1024) \
1048                         { \
1049                           a->busy_##n##_ports--; \
1050                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1051                         } \
1052                       break;
1053                       foreach_snat_protocol
1054 #undef _
1055                     default:
1056                       nat_elog_info ("unknown protocol");
1057                       return VNET_API_ERROR_INVALID_VALUE_2;
1058                     }
1059                   break;
1060                 }
1061             }
1062         }
1063
1064       if (sm->num_workers > 1)
1065         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1066       else
1067         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1068
1069       m_key.addr = m->local_addr;
1070       m_key.port = m->local_port;
1071       m_key.protocol = m->proto;
1072       m_key.fib_index = fib_index;
1073       kv.key = m_key.as_u64;
1074       if (!out2in_only)
1075         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1076
1077       /* Delete session(s) for static mapping if exist */
1078       if (!(sm->static_mapping_only) ||
1079           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1080         {
1081           u_key.addr = m->local_addr;
1082           u_key.fib_index = fib_index;
1083           kv.key = u_key.as_u64;
1084           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1085             {
1086               user_index = value.value;
1087               u = pool_elt_at_index (tsm->users, user_index);
1088               if (u->nstaticsessions)
1089                 {
1090                   head_index = u->sessions_per_user_list_head_index;
1091                   head = pool_elt_at_index (tsm->list_pool, head_index);
1092                   elt_index = head->next;
1093                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1094                   ses_index = elt->value;
1095                   while (ses_index != ~0)
1096                     {
1097                       s = pool_elt_at_index (tsm->sessions, ses_index);
1098                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1099                       ses_index = elt->value;
1100
1101                       if (!addr_only)
1102                         {
1103                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1104                               (clib_net_to_host_u16 (s->out2in.port) !=
1105                                e_port))
1106                             continue;
1107                         }
1108
1109                       if (is_lb_session (s))
1110                         continue;
1111
1112                       if (!snat_is_session_static (s))
1113                         continue;
1114
1115                       nat_free_session_data (sm, s,
1116                                              tsm - sm->per_thread_data, 0);
1117                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1118
1119                       if (!addr_only && !sm->endpoint_dependent)
1120                         break;
1121                     }
1122                 }
1123             }
1124         }
1125
1126       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1127       if (pool_elts (m->locals))
1128         return 0;
1129
1130       m_key.addr = m->external_addr;
1131       m_key.port = m->external_port;
1132       m_key.fib_index = 0;
1133       kv.key = m_key.as_u64;
1134       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1135
1136       vec_free (m->tag);
1137       vec_free (m->workers);
1138       /* Delete static mapping from pool */
1139       pool_put (sm->static_mappings, m);
1140     }
1141
1142   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1143     return 0;
1144
1145   /* Add/delete external address to FIB */
1146   /* *INDENT-OFF* */
1147   pool_foreach (interface, sm->interfaces,
1148   ({
1149     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1150       continue;
1151
1152     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1153     break;
1154   }));
1155   pool_foreach (interface, sm->output_feature_interfaces,
1156   ({
1157     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1158       continue;
1159
1160     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1161     break;
1162   }));
1163   /* *INDENT-ON* */
1164
1165   return 0;
1166 }
1167
1168 int
1169 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1170                                  snat_protocol_t proto,
1171                                  nat44_lb_addr_port_t * locals, u8 is_add,
1172                                  twice_nat_type_t twice_nat, u8 out2in_only,
1173                                  u8 * tag, u32 affinity)
1174 {
1175   snat_main_t *sm = &snat_main;
1176   snat_static_mapping_t *m;
1177   snat_session_key_t m_key;
1178   clib_bihash_kv_8_8_t kv, value;
1179   snat_address_t *a = 0;
1180   int i;
1181   nat44_lb_addr_port_t *local;
1182   u32 elt_index, head_index, ses_index;
1183   snat_main_per_thread_data_t *tsm;
1184   snat_user_key_t u_key;
1185   snat_user_t *u;
1186   snat_session_t *s;
1187   dlist_elt_t *head, *elt;
1188   uword *bitmap = 0;
1189
1190   if (!sm->endpoint_dependent)
1191     return VNET_API_ERROR_FEATURE_DISABLED;
1192
1193   m_key.addr = e_addr;
1194   m_key.port = e_port;
1195   m_key.protocol = proto;
1196   m_key.fib_index = 0;
1197   kv.key = m_key.as_u64;
1198   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1199     m = 0;
1200   else
1201     m = pool_elt_at_index (sm->static_mappings, value.value);
1202
1203   if (is_add)
1204     {
1205       if (m)
1206         return VNET_API_ERROR_VALUE_EXIST;
1207
1208       if (vec_len (locals) < 2)
1209         return VNET_API_ERROR_INVALID_VALUE;
1210
1211       /* Find external address in allocated addresses and reserve port for
1212          address and port pair mapping when dynamic translations enabled */
1213       if (!(sm->static_mapping_only || out2in_only))
1214         {
1215           for (i = 0; i < vec_len (sm->addresses); i++)
1216             {
1217               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1218                 {
1219                   a = sm->addresses + i;
1220                   /* External port must be unused */
1221                   switch (proto)
1222                     {
1223 #define _(N, j, n, s) \
1224                     case SNAT_PROTOCOL_##N: \
1225                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1226                         return VNET_API_ERROR_INVALID_VALUE; \
1227                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1228                       if (e_port > 1024) \
1229                         { \
1230                           a->busy_##n##_ports++; \
1231                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1232                         } \
1233                       break;
1234                       foreach_snat_protocol
1235 #undef _
1236                     default:
1237                       nat_elog_info ("unknown protocol");
1238                       return VNET_API_ERROR_INVALID_VALUE_2;
1239                     }
1240                   break;
1241                 }
1242             }
1243           /* External address must be allocated */
1244           if (!a)
1245             return VNET_API_ERROR_NO_SUCH_ENTRY;
1246         }
1247
1248       pool_get (sm->static_mappings, m);
1249       clib_memset (m, 0, sizeof (*m));
1250       m->tag = vec_dup (tag);
1251       m->external_addr = e_addr;
1252       m->external_port = e_port;
1253       m->proto = proto;
1254       m->twice_nat = twice_nat;
1255       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1256       if (out2in_only)
1257         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1258       m->affinity = affinity;
1259
1260       if (affinity)
1261         m->affinity_per_service_list_head_index =
1262           nat_affinity_get_per_service_list_head_index ();
1263       else
1264         m->affinity_per_service_list_head_index = ~0;
1265
1266       m_key.addr = m->external_addr;
1267       m_key.port = m->external_port;
1268       m_key.protocol = m->proto;
1269       m_key.fib_index = 0;
1270       kv.key = m_key.as_u64;
1271       kv.value = m - sm->static_mappings;
1272       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1273         {
1274           nat_elog_err ("static_mapping_by_external key add failed");
1275           return VNET_API_ERROR_UNSPECIFIED;
1276         }
1277
1278       m_key.fib_index = m->fib_index;
1279       for (i = 0; i < vec_len (locals); i++)
1280         {
1281           locals[i].fib_index =
1282             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1283                                                locals[i].vrf_id,
1284                                                FIB_SOURCE_PLUGIN_LOW);
1285           m_key.addr = locals[i].addr;
1286           m_key.fib_index = locals[i].fib_index;
1287           if (!out2in_only)
1288             {
1289               m_key.port = locals[i].port;
1290               kv.key = m_key.as_u64;
1291               kv.value = m - sm->static_mappings;
1292               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1293             }
1294           locals[i].prefix = (i == 0) ? locals[i].probability :
1295             (locals[i - 1].prefix + locals[i].probability);
1296           pool_get (m->locals, local);
1297           *local = locals[i];
1298           if (sm->num_workers > 1)
1299             {
1300               ip4_header_t ip = {
1301                 .src_address = locals[i].addr,
1302               };
1303               bitmap =
1304                 clib_bitmap_set (bitmap,
1305                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1306                                  1);
1307             }
1308         }
1309
1310       /* Assign workers */
1311       if (sm->num_workers > 1)
1312         {
1313           /* *INDENT-OFF* */
1314           clib_bitmap_foreach (i, bitmap,
1315             ({
1316                vec_add1(m->workers, i);
1317             }));
1318           /* *INDENT-ON* */
1319         }
1320     }
1321   else
1322     {
1323       if (!m)
1324         return VNET_API_ERROR_NO_SUCH_ENTRY;
1325
1326       if (!is_lb_static_mapping (m))
1327         return VNET_API_ERROR_INVALID_VALUE;
1328
1329       /* Free external address port */
1330       if (!(sm->static_mapping_only || out2in_only))
1331         {
1332           for (i = 0; i < vec_len (sm->addresses); i++)
1333             {
1334               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1335                 {
1336                   a = sm->addresses + i;
1337                   switch (proto)
1338                     {
1339 #define _(N, j, n, s) \
1340                     case SNAT_PROTOCOL_##N: \
1341                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1342                       if (e_port > 1024) \
1343                         { \
1344                           a->busy_##n##_ports--; \
1345                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1346                         } \
1347                       break;
1348                       foreach_snat_protocol
1349 #undef _
1350                     default:
1351                       nat_elog_info ("unknown protocol");
1352                       return VNET_API_ERROR_INVALID_VALUE_2;
1353                     }
1354                   break;
1355                 }
1356             }
1357         }
1358
1359       m_key.addr = m->external_addr;
1360       m_key.port = m->external_port;
1361       m_key.protocol = m->proto;
1362       m_key.fib_index = 0;
1363       kv.key = m_key.as_u64;
1364       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1365         {
1366           nat_elog_err ("static_mapping_by_external key del failed");
1367           return VNET_API_ERROR_UNSPECIFIED;
1368         }
1369
1370       /* *INDENT-OFF* */
1371       pool_foreach (local, m->locals,
1372       ({
1373           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1374                             FIB_SOURCE_PLUGIN_LOW);
1375           m_key.addr = local->addr;
1376           if (!out2in_only)
1377             {
1378               m_key.port = local->port;
1379               m_key.fib_index = local->fib_index;
1380               kv.key = m_key.as_u64;
1381               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1382                 {
1383                   nat_elog_err ("static_mapping_by_local key del failed");
1384                   return VNET_API_ERROR_UNSPECIFIED;
1385                 }
1386             }
1387
1388           if (sm->num_workers > 1)
1389             {
1390               ip4_header_t ip = {
1391                 .src_address = local->addr,
1392               };
1393               tsm = vec_elt_at_index (sm->per_thread_data,
1394                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1395             }
1396           else
1397             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1398
1399           /* Delete sessions */
1400           u_key.addr = local->addr;
1401           u_key.fib_index = local->fib_index;
1402           kv.key = u_key.as_u64;
1403           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1404             {
1405               u = pool_elt_at_index (tsm->users, value.value);
1406               if (u->nstaticsessions)
1407                 {
1408                   head_index = u->sessions_per_user_list_head_index;
1409                   head = pool_elt_at_index (tsm->list_pool, head_index);
1410                   elt_index = head->next;
1411                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1412                   ses_index = elt->value;
1413                   while (ses_index != ~0)
1414                     {
1415                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1416                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1417                       ses_index = elt->value;
1418
1419                       if (!(is_lb_session (s)))
1420                         continue;
1421
1422                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1423                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1424                         continue;
1425
1426                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1427                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1428                     }
1429                 }
1430             }
1431       }));
1432       /* *INDENT-ON* */
1433       if (m->affinity)
1434         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1435       pool_free (m->locals);
1436       vec_free (m->tag);
1437       vec_free (m->workers);
1438
1439       pool_put (sm->static_mappings, m);
1440     }
1441
1442   return 0;
1443 }
1444
1445 int
1446 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1447                                        ip4_address_t l_addr, u16 l_port,
1448                                        snat_protocol_t proto, u32 vrf_id,
1449                                        u8 probability, u8 is_add)
1450 {
1451   snat_main_t *sm = &snat_main;
1452   snat_static_mapping_t *m = 0;
1453   snat_session_key_t m_key;
1454   clib_bihash_kv_8_8_t kv, value;
1455   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1456   snat_main_per_thread_data_t *tsm;
1457   snat_user_key_t u_key;
1458   snat_user_t *u;
1459   snat_session_t *s;
1460   dlist_elt_t *head, *elt;
1461   u32 elt_index, head_index, ses_index, *locals = 0;
1462   uword *bitmap = 0;
1463   int i;
1464
1465   if (!sm->endpoint_dependent)
1466     return VNET_API_ERROR_FEATURE_DISABLED;
1467
1468   m_key.addr = e_addr;
1469   m_key.port = e_port;
1470   m_key.protocol = proto;
1471   m_key.fib_index = 0;
1472   kv.key = m_key.as_u64;
1473   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1474     m = pool_elt_at_index (sm->static_mappings, value.value);
1475
1476   if (!m)
1477     return VNET_API_ERROR_NO_SUCH_ENTRY;
1478
1479   if (!is_lb_static_mapping (m))
1480     return VNET_API_ERROR_INVALID_VALUE;
1481
1482   /* *INDENT-OFF* */
1483   pool_foreach (local, m->locals,
1484   ({
1485     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1486         (local->vrf_id == vrf_id))
1487       {
1488         match_local = local;
1489         break;
1490       }
1491   }));
1492   /* *INDENT-ON* */
1493
1494   if (is_add)
1495     {
1496       if (match_local)
1497         return VNET_API_ERROR_VALUE_EXIST;
1498
1499       pool_get (m->locals, local);
1500       clib_memset (local, 0, sizeof (*local));
1501       local->addr.as_u32 = l_addr.as_u32;
1502       local->port = l_port;
1503       local->probability = probability;
1504       local->vrf_id = vrf_id;
1505       local->fib_index =
1506         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1507                                            FIB_SOURCE_PLUGIN_LOW);
1508
1509       if (!is_out2in_only_static_mapping (m))
1510         {
1511           m_key.addr = l_addr;
1512           m_key.port = l_port;
1513           m_key.fib_index = local->fib_index;
1514           kv.key = m_key.as_u64;
1515           kv.value = m - sm->static_mappings;
1516           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1517             nat_elog_err ("static_mapping_by_local key add failed");
1518         }
1519     }
1520   else
1521     {
1522       if (!match_local)
1523         return VNET_API_ERROR_NO_SUCH_ENTRY;
1524
1525       if (pool_elts (m->locals) < 3)
1526         return VNET_API_ERROR_UNSPECIFIED;
1527
1528       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1529                         FIB_SOURCE_PLUGIN_LOW);
1530
1531       if (!is_out2in_only_static_mapping (m))
1532         {
1533           m_key.addr = l_addr;
1534           m_key.port = l_port;
1535           m_key.fib_index = match_local->fib_index;
1536           kv.key = m_key.as_u64;
1537           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1538             nat_elog_err ("static_mapping_by_local key del failed");
1539         }
1540
1541       if (sm->num_workers > 1)
1542         {
1543           ip4_header_t ip = {
1544             .src_address = local->addr,
1545           };
1546           tsm = vec_elt_at_index (sm->per_thread_data,
1547                                   sm->worker_in2out_cb (&ip, m->fib_index,
1548                                                         0));
1549         }
1550       else
1551         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1552
1553       /* Delete sessions */
1554       u_key.addr = match_local->addr;
1555       u_key.fib_index = match_local->fib_index;
1556       kv.key = u_key.as_u64;
1557       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1558         {
1559           u = pool_elt_at_index (tsm->users, value.value);
1560           if (u->nstaticsessions)
1561             {
1562               head_index = u->sessions_per_user_list_head_index;
1563               head = pool_elt_at_index (tsm->list_pool, head_index);
1564               elt_index = head->next;
1565               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1566               ses_index = elt->value;
1567               while (ses_index != ~0)
1568                 {
1569                   s = pool_elt_at_index (tsm->sessions, ses_index);
1570                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1571                   ses_index = elt->value;
1572
1573                   if (!(is_lb_session (s)))
1574                     continue;
1575
1576                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1577                       (clib_net_to_host_u16 (s->in2out.port) !=
1578                        match_local->port))
1579                     continue;
1580
1581                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1582                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1583                 }
1584             }
1585         }
1586
1587       pool_put (m->locals, match_local);
1588     }
1589
1590   vec_free (m->workers);
1591
1592   /* *INDENT-OFF* */
1593   pool_foreach (local, m->locals,
1594   ({
1595     vec_add1 (locals, local - m->locals);
1596     if (sm->num_workers > 1)
1597       {
1598         ip4_header_t ip;
1599         ip.src_address.as_u32 = local->addr.as_u32,
1600         bitmap = clib_bitmap_set (bitmap,
1601                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1602                                   1);
1603       }
1604   }));
1605   /* *INDENT-ON* */
1606
1607   ASSERT (vec_len (locals) > 1);
1608
1609   local = pool_elt_at_index (m->locals, locals[0]);
1610   local->prefix = local->probability;
1611   for (i = 1; i < vec_len (locals); i++)
1612     {
1613       local = pool_elt_at_index (m->locals, locals[i]);
1614       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1615       local->prefix = local->probability + prev_local->prefix;
1616     }
1617
1618   /* Assign workers */
1619   if (sm->num_workers > 1)
1620     {
1621       /* *INDENT-OFF* */
1622       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1623       /* *INDENT-ON* */
1624     }
1625
1626   return 0;
1627 }
1628
1629 int
1630 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1631                   u8 twice_nat)
1632 {
1633   snat_address_t *a = 0;
1634   snat_session_t *ses;
1635   u32 *ses_to_be_removed = 0, *ses_index;
1636   snat_main_per_thread_data_t *tsm;
1637   snat_static_mapping_t *m;
1638   snat_interface_t *interface;
1639   int i;
1640   snat_address_t *addresses =
1641     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1642
1643   /* Find SNAT address */
1644   for (i = 0; i < vec_len (addresses); i++)
1645     {
1646       if (addresses[i].addr.as_u32 == addr.as_u32)
1647         {
1648           a = addresses + i;
1649           break;
1650         }
1651     }
1652   if (!a)
1653     return VNET_API_ERROR_NO_SUCH_ENTRY;
1654
1655   if (delete_sm)
1656     {
1657       /* *INDENT-OFF* */
1658       pool_foreach (m, sm->static_mappings,
1659       ({
1660           if (m->external_addr.as_u32 == addr.as_u32)
1661             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1662                                             m->local_port, m->external_port,
1663                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1664                                             m->proto, 0, m->twice_nat,
1665                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1666       }));
1667       /* *INDENT-ON* */
1668     }
1669   else
1670     {
1671       /* Check if address is used in some static mapping */
1672       if (is_snat_address_used_in_static_mapping (sm, addr))
1673         {
1674           nat_elog_notice ("address used in static mapping");
1675           return VNET_API_ERROR_UNSPECIFIED;
1676         }
1677     }
1678
1679   if (a->fib_index != ~0)
1680     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1681
1682   /* Delete sessions using address */
1683   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1684     {
1685       /* *INDENT-OFF* */
1686       vec_foreach (tsm, sm->per_thread_data)
1687         {
1688           pool_foreach (ses, tsm->sessions, ({
1689             if (ses->out2in.addr.as_u32 == addr.as_u32)
1690               {
1691                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1692                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1693               }
1694           }));
1695
1696           vec_foreach (ses_index, ses_to_be_removed)
1697             {
1698               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1699               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1700             }
1701
1702           vec_free (ses_to_be_removed);
1703         }
1704       /* *INDENT-ON* */
1705     }
1706
1707 #define _(N, i, n, s) \
1708   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1709   vec_free (a->busy_##n##_ports_per_thread);
1710   foreach_snat_protocol
1711 #undef _
1712     if (twice_nat)
1713     {
1714       vec_del1 (sm->twice_nat_addresses, i);
1715       return 0;
1716     }
1717   else
1718     vec_del1 (sm->addresses, i);
1719
1720   /* Delete external address from FIB */
1721   /* *INDENT-OFF* */
1722   pool_foreach (interface, sm->interfaces,
1723   ({
1724     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1725       continue;
1726
1727     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1728     break;
1729   }));
1730   pool_foreach (interface, sm->output_feature_interfaces,
1731   ({
1732     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1733       continue;
1734
1735     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1736     break;
1737   }));
1738   /* *INDENT-ON* */
1739
1740   return 0;
1741 }
1742
1743 int
1744 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1745 {
1746   snat_main_t *sm = &snat_main;
1747   snat_interface_t *i;
1748   const char *feature_name, *del_feature_name;
1749   snat_address_t *ap;
1750   snat_static_mapping_t *m;
1751   snat_det_map_t *dm;
1752   nat_outside_fib_t *outside_fib;
1753   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1754                                                        sw_if_index);
1755
1756   if (sm->out2in_dpo && !is_inside)
1757     return VNET_API_ERROR_UNSUPPORTED;
1758
1759   /* *INDENT-OFF* */
1760   pool_foreach (i, sm->output_feature_interfaces,
1761   ({
1762     if (i->sw_if_index == sw_if_index)
1763       return VNET_API_ERROR_VALUE_EXIST;
1764   }));
1765   /* *INDENT-ON* */
1766
1767   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1768     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1769   else
1770     {
1771       if (sm->num_workers > 1 && !sm->deterministic)
1772         feature_name =
1773           is_inside ? "nat44-in2out-worker-handoff" :
1774           "nat44-out2in-worker-handoff";
1775       else if (sm->deterministic)
1776         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1777       else if (sm->endpoint_dependent)
1778         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1779       else
1780         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1781     }
1782
1783   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1784     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1785                                                       NAT_FQ_NELTS);
1786
1787   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1788     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1789                                                       NAT_FQ_NELTS);
1790
1791   if (!is_inside)
1792     {
1793       /* *INDENT-OFF* */
1794       vec_foreach (outside_fib, sm->outside_fibs)
1795         {
1796           if (outside_fib->fib_index == fib_index)
1797             {
1798               if (is_del)
1799                 {
1800                   outside_fib->refcount--;
1801                   if (!outside_fib->refcount)
1802                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1803                 }
1804               else
1805                 outside_fib->refcount++;
1806               goto feature_set;
1807             }
1808         }
1809       /* *INDENT-ON* */
1810       if (!is_del)
1811         {
1812           vec_add2 (sm->outside_fibs, outside_fib, 1);
1813           outside_fib->refcount = 1;
1814           outside_fib->fib_index = fib_index;
1815         }
1816     }
1817 feature_set:
1818   /* *INDENT-OFF* */
1819   pool_foreach (i, sm->interfaces,
1820   ({
1821     if (i->sw_if_index == sw_if_index)
1822       {
1823         if (is_del)
1824           {
1825             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1826               {
1827                 if (is_inside)
1828                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1829                 else
1830                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1831
1832                 if (sm->num_workers > 1 && !sm->deterministic)
1833                   {
1834                     del_feature_name = "nat44-handoff-classify";
1835                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1836                                                  "nat44-out2in-worker-handoff";
1837                   }
1838                 else if (sm->deterministic)
1839                   {
1840                     del_feature_name = "nat44-det-classify";
1841                     feature_name = !is_inside ?  "nat44-det-in2out" :
1842                                                  "nat44-det-out2in";
1843                   }
1844                 else if (sm->endpoint_dependent)
1845                   {
1846                     del_feature_name = "nat44-ed-classify";
1847                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1848                                                  "nat44-ed-out2in";
1849                   }
1850                 else
1851                   {
1852                     del_feature_name = "nat44-classify";
1853                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1854                   }
1855
1856                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1857                                              sw_if_index, 0, 0, 0);
1858                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1859                                              sw_if_index, 1, 0, 0);
1860                 if (!is_inside)
1861                   {
1862                     if (sm->endpoint_dependent)
1863                       vnet_feature_enable_disable ("ip4-local",
1864                                                    "nat44-ed-hairpinning",
1865                                                    sw_if_index, 1, 0, 0);
1866                     else if (!sm->deterministic)
1867                       vnet_feature_enable_disable ("ip4-local",
1868                                                    "nat44-hairpinning",
1869                                                    sw_if_index, 1, 0, 0);
1870                   }
1871               }
1872             else
1873               {
1874                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1875                                              sw_if_index, 0, 0, 0);
1876                 pool_put (sm->interfaces, i);
1877                 if (is_inside)
1878                   {
1879                     if (sm->endpoint_dependent)
1880                       vnet_feature_enable_disable ("ip4-local",
1881                                                    "nat44-ed-hairpinning",
1882                                                    sw_if_index, 0, 0, 0);
1883                     else if (!sm->deterministic)
1884                       vnet_feature_enable_disable ("ip4-local",
1885                                                    "nat44-hairpinning",
1886                                                    sw_if_index, 0, 0, 0);
1887                   }
1888               }
1889           }
1890         else
1891           {
1892             if ((nat_interface_is_inside(i) && is_inside) ||
1893                 (nat_interface_is_outside(i) && !is_inside))
1894               return 0;
1895
1896             if (sm->num_workers > 1 && !sm->deterministic)
1897               {
1898                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1899                                                  "nat44-out2in-worker-handoff";
1900                 feature_name = "nat44-handoff-classify";
1901               }
1902             else if (sm->deterministic)
1903               {
1904                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1905                                                  "nat44-det-out2in";
1906                 feature_name = "nat44-det-classify";
1907               }
1908             else if (sm->endpoint_dependent)
1909               {
1910                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1911                                                  "nat44-ed-out2in";
1912                 feature_name = "nat44-ed-classify";
1913               }
1914             else
1915               {
1916                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1917                 feature_name = "nat44-classify";
1918               }
1919
1920             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1921                                          sw_if_index, 0, 0, 0);
1922             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1923                                          sw_if_index, 1, 0, 0);
1924             if (!is_inside)
1925               {
1926                 if (sm->endpoint_dependent)
1927                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1928                                                sw_if_index, 0, 0, 0);
1929                 else if (!sm->deterministic)
1930                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1931                                                sw_if_index, 0, 0, 0);
1932               }
1933             goto set_flags;
1934           }
1935
1936         goto fib;
1937       }
1938   }));
1939   /* *INDENT-ON* */
1940
1941   if (is_del)
1942     return VNET_API_ERROR_NO_SUCH_ENTRY;
1943
1944   pool_get (sm->interfaces, i);
1945   i->sw_if_index = sw_if_index;
1946   i->flags = 0;
1947   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1948                                0);
1949
1950   if (is_inside && !sm->out2in_dpo)
1951     {
1952       if (sm->endpoint_dependent)
1953         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1954                                      sw_if_index, 1, 0, 0);
1955       else if (!sm->deterministic)
1956         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1957                                      sw_if_index, 1, 0, 0);
1958     }
1959
1960 set_flags:
1961   if (is_inside)
1962     {
1963       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1964       return 0;
1965     }
1966   else
1967     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1968
1969   /* Add/delete external addresses to FIB */
1970 fib:
1971   /* *INDENT-OFF* */
1972   vec_foreach (ap, sm->addresses)
1973     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1974
1975   pool_foreach (m, sm->static_mappings,
1976   ({
1977     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1978       continue;
1979
1980     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1981   }));
1982
1983   pool_foreach (dm, sm->det_maps,
1984   ({
1985     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1986   }));
1987   /* *INDENT-ON* */
1988
1989   return 0;
1990 }
1991
1992 int
1993 snat_interface_add_del_output_feature (u32 sw_if_index,
1994                                        u8 is_inside, int is_del)
1995 {
1996   snat_main_t *sm = &snat_main;
1997   snat_interface_t *i;
1998   snat_address_t *ap;
1999   snat_static_mapping_t *m;
2000   nat_outside_fib_t *outside_fib;
2001   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2002                                                        sw_if_index);
2003
2004
2005   if (sm->deterministic ||
2006       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2007     return VNET_API_ERROR_UNSUPPORTED;
2008
2009   /* *INDENT-OFF* */
2010   pool_foreach (i, sm->interfaces,
2011   ({
2012     if (i->sw_if_index == sw_if_index)
2013       return VNET_API_ERROR_VALUE_EXIST;
2014   }));
2015   /* *INDENT-ON* */
2016
2017   if (!is_inside)
2018     {
2019       /* *INDENT-OFF* */
2020       vec_foreach (outside_fib, sm->outside_fibs)
2021         {
2022           if (outside_fib->fib_index == fib_index)
2023             {
2024               if (is_del)
2025                 {
2026                   outside_fib->refcount--;
2027                   if (!outside_fib->refcount)
2028                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2029                 }
2030               else
2031                 outside_fib->refcount++;
2032               goto feature_set;
2033             }
2034         }
2035       /* *INDENT-ON* */
2036       if (!is_del)
2037         {
2038           vec_add2 (sm->outside_fibs, outside_fib, 1);
2039           outside_fib->refcount = 1;
2040           outside_fib->fib_index = fib_index;
2041         }
2042     }
2043
2044 feature_set:
2045   if (is_inside)
2046     {
2047       if (sm->endpoint_dependent)
2048         {
2049           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2050                                        sw_if_index, !is_del, 0, 0);
2051           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2052                                        sw_if_index, !is_del, 0, 0);
2053         }
2054       else
2055         {
2056           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2057                                        sw_if_index, !is_del, 0, 0);
2058           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2059                                        sw_if_index, !is_del, 0, 0);
2060         }
2061       goto fq;
2062     }
2063
2064   if (sm->num_workers > 1)
2065     {
2066       vnet_feature_enable_disable ("ip4-unicast",
2067                                    "nat44-out2in-worker-handoff",
2068                                    sw_if_index, !is_del, 0, 0);
2069       vnet_feature_enable_disable ("ip4-output",
2070                                    "nat44-in2out-output-worker-handoff",
2071                                    sw_if_index, !is_del, 0, 0);
2072     }
2073   else
2074     {
2075       if (sm->endpoint_dependent)
2076         {
2077           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2078                                        sw_if_index, !is_del, 0, 0);
2079           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2080                                        sw_if_index, !is_del, 0, 0);
2081         }
2082       else
2083         {
2084           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2085                                        sw_if_index, !is_del, 0, 0);
2086           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2087                                        sw_if_index, !is_del, 0, 0);
2088         }
2089     }
2090
2091 fq:
2092   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2093     sm->fq_in2out_output_index =
2094       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2095
2096   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2097     sm->fq_out2in_index =
2098       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2099
2100   /* *INDENT-OFF* */
2101   pool_foreach (i, sm->output_feature_interfaces,
2102   ({
2103     if (i->sw_if_index == sw_if_index)
2104       {
2105         if (is_del)
2106           pool_put (sm->output_feature_interfaces, i);
2107         else
2108           return VNET_API_ERROR_VALUE_EXIST;
2109
2110         goto fib;
2111       }
2112   }));
2113   /* *INDENT-ON* */
2114
2115   if (is_del)
2116     return VNET_API_ERROR_NO_SUCH_ENTRY;
2117
2118   pool_get (sm->output_feature_interfaces, i);
2119   i->sw_if_index = sw_if_index;
2120   i->flags = 0;
2121   if (is_inside)
2122     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2123   else
2124     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2125
2126   /* Add/delete external addresses to FIB */
2127 fib:
2128   if (is_inside)
2129     return 0;
2130
2131   /* *INDENT-OFF* */
2132   vec_foreach (ap, sm->addresses)
2133     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2134
2135   pool_foreach (m, sm->static_mappings,
2136   ({
2137     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2138       continue;
2139
2140     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2141   }));
2142   /* *INDENT-ON* */
2143
2144   return 0;
2145 }
2146
2147 int
2148 snat_set_workers (uword * bitmap)
2149 {
2150   snat_main_t *sm = &snat_main;
2151   int i, j = 0;
2152
2153   if (sm->num_workers < 2)
2154     return VNET_API_ERROR_FEATURE_DISABLED;
2155
2156   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2157     return VNET_API_ERROR_INVALID_WORKER;
2158
2159   vec_free (sm->workers);
2160   /* *INDENT-OFF* */
2161   clib_bitmap_foreach (i, bitmap,
2162     ({
2163       vec_add1(sm->workers, i);
2164       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2165       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2166       j++;
2167     }));
2168   /* *INDENT-ON* */
2169
2170   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2171   sm->num_snat_thread = _vec_len (sm->workers);
2172
2173   return 0;
2174 }
2175
2176 static void
2177 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2178                          u32 old_fib_index)
2179 {
2180   snat_main_t *sm = &snat_main;
2181   nat_outside_fib_t *outside_fib;
2182   snat_interface_t *i;
2183   u8 is_add = 1;
2184   u8 match = 0;
2185
2186   if (new_fib_index == old_fib_index)
2187     return;
2188
2189   if (!vec_len (sm->outside_fibs))
2190     return;
2191
2192   /* *INDENT-OFF* */
2193   pool_foreach (i, sm->interfaces,
2194     ({
2195       if (i->sw_if_index == sw_if_index)
2196         {
2197           if (!(nat_interface_is_outside (i)))
2198             return;
2199           match = 1;
2200         }
2201     }));
2202
2203   pool_foreach (i, sm->output_feature_interfaces,
2204     ({
2205       if (i->sw_if_index == sw_if_index)
2206         {
2207           if (!(nat_interface_is_outside (i)))
2208             return;
2209           match = 1;
2210         }
2211     }));
2212   /* *INDENT-ON* */
2213
2214   if (!match)
2215     return;
2216
2217   vec_foreach (outside_fib, sm->outside_fibs)
2218   {
2219     if (outside_fib->fib_index == old_fib_index)
2220       {
2221         outside_fib->refcount--;
2222         if (!outside_fib->refcount)
2223           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2224         break;
2225       }
2226   }
2227
2228   vec_foreach (outside_fib, sm->outside_fibs)
2229   {
2230     if (outside_fib->fib_index == new_fib_index)
2231       {
2232         outside_fib->refcount++;
2233         is_add = 0;
2234         break;
2235       }
2236   }
2237
2238   if (is_add)
2239     {
2240       vec_add2 (sm->outside_fibs, outside_fib, 1);
2241       outside_fib->refcount = 1;
2242       outside_fib->fib_index = new_fib_index;
2243     }
2244 }
2245
2246 static void
2247 snat_ip4_table_bind (ip4_main_t * im,
2248                      uword opaque,
2249                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2250 {
2251   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2252 }
2253
2254 static void
2255 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2256                                        uword opaque,
2257                                        u32 sw_if_index,
2258                                        ip4_address_t * address,
2259                                        u32 address_length,
2260                                        u32 if_address_index, u32 is_delete);
2261
2262 static void
2263 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2264                                  uword opaque,
2265                                  u32 sw_if_index,
2266                                  ip4_address_t * address,
2267                                  u32 address_length,
2268                                  u32 if_address_index, u32 is_delete);
2269
2270 static int
2271 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2272                                  u32 fib_index,
2273                                  u32 thread_index,
2274                                  snat_session_key_t * k,
2275                                  u16 port_per_thread, u32 snat_thread_index);
2276
2277 static clib_error_t *
2278 snat_init (vlib_main_t * vm)
2279 {
2280   snat_main_t *sm = &snat_main;
2281   clib_error_t *error = 0;
2282   ip4_main_t *im = &ip4_main;
2283   ip_lookup_main_t *lm = &im->lookup_main;
2284   uword *p;
2285   vlib_thread_registration_t *tr;
2286   vlib_thread_main_t *tm = vlib_get_thread_main ();
2287   uword *bitmap = 0;
2288   u32 i;
2289   ip4_add_del_interface_address_callback_t cb4;
2290   vlib_node_t *node;
2291
2292   sm->vlib_main = vm;
2293   sm->vnet_main = vnet_get_main ();
2294   sm->ip4_main = im;
2295   sm->ip4_lookup_main = lm;
2296   sm->api_main = &api_main;
2297   sm->first_worker_index = 0;
2298   sm->num_workers = 0;
2299   sm->num_snat_thread = 1;
2300   sm->workers = 0;
2301   sm->port_per_thread = 0xffff - 1024;
2302   sm->fq_in2out_index = ~0;
2303   sm->fq_in2out_output_index = ~0;
2304   sm->fq_out2in_index = ~0;
2305   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2306   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2307   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2308   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2309   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2310   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2311   sm->forwarding_enabled = 0;
2312   sm->log_class = vlib_log_register_class ("nat", 0);
2313   sm->log_level = SNAT_LOG_ERROR;
2314   sm->mss_clamping = 0;
2315
2316   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2317   sm->error_node_index = node->index;
2318
2319   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2320   sm->in2out_node_index = node->index;
2321   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2322   sm->in2out_output_node_index = node->index;
2323   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2324   sm->in2out_fast_node_index = node->index;
2325   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2326   sm->in2out_slowpath_node_index = node->index;
2327   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2328   sm->in2out_slowpath_output_node_index = node->index;
2329   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-reass");
2330   sm->in2out_reass_node_index = node->index;
2331
2332   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2333   sm->ed_in2out_node_index = node->index;
2334   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2335   sm->ed_in2out_slowpath_node_index = node->index;
2336   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-reass");
2337   sm->ed_in2out_reass_node_index = node->index;
2338
2339   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2340   sm->out2in_node_index = node->index;
2341   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2342   sm->out2in_fast_node_index = node->index;
2343   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-reass");
2344   sm->out2in_reass_node_index = node->index;
2345
2346   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2347   sm->ed_out2in_node_index = node->index;
2348   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2349   sm->ed_out2in_slowpath_node_index = node->index;
2350   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-reass");
2351   sm->ed_out2in_reass_node_index = node->index;
2352
2353   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2354   sm->det_in2out_node_index = node->index;
2355   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2356   sm->det_out2in_node_index = node->index;
2357
2358   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2359   sm->hairpinning_node_index = node->index;
2360   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2361   sm->hairpin_dst_node_index = node->index;
2362   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2363   sm->hairpin_src_node_index = node->index;
2364   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2365   sm->ed_hairpinning_node_index = node->index;
2366   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2367   sm->ed_hairpin_dst_node_index = node->index;
2368   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2369   sm->ed_hairpin_src_node_index = node->index;
2370
2371   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2372   if (p)
2373     {
2374       tr = (vlib_thread_registration_t *) p[0];
2375       if (tr)
2376         {
2377           sm->num_workers = tr->count;
2378           sm->first_worker_index = tr->first_index;
2379         }
2380     }
2381
2382   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2383
2384   /* Use all available workers by default */
2385   if (sm->num_workers > 1)
2386     {
2387       for (i = 0; i < sm->num_workers; i++)
2388         bitmap = clib_bitmap_set (bitmap, i, 1);
2389       snat_set_workers (bitmap);
2390       clib_bitmap_free (bitmap);
2391     }
2392   else
2393     {
2394       sm->per_thread_data[0].snat_thread_index = 0;
2395     }
2396
2397   error = snat_api_init (vm, sm);
2398   if (error)
2399     return error;
2400
2401   /* Set up the interface address add/del callback */
2402   cb4.function = snat_ip4_add_del_interface_address_cb;
2403   cb4.function_opaque = 0;
2404
2405   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2406
2407   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2408   cb4.function_opaque = 0;
2409
2410   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2411
2412   nat_dpo_module_init ();
2413
2414   /* Init counters */
2415   sm->total_users.name = "total-users";
2416   sm->total_users.stat_segment_name = "/nat44/total-users";
2417   vlib_validate_simple_counter (&sm->total_users, 0);
2418   vlib_zero_simple_counter (&sm->total_users, 0);
2419   sm->total_sessions.name = "total-sessions";
2420   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2421   vlib_validate_simple_counter (&sm->total_sessions, 0);
2422   vlib_zero_simple_counter (&sm->total_sessions, 0);
2423
2424   /* Init IPFIX logging */
2425   snat_ipfix_logging_init (vm);
2426
2427   /* Init NAT64 */
2428   error = nat64_init (vm);
2429   if (error)
2430     return error;
2431
2432   dslite_init (vm);
2433
2434   nat66_init (vm);
2435
2436   ip4_table_bind_callback_t cbt4 = {
2437     .function = snat_ip4_table_bind,
2438   };
2439   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2440
2441   /* Init virtual fragmenentation reassembly */
2442   return nat_reass_init (vm);
2443 }
2444
2445 VLIB_INIT_FUNCTION (snat_init);
2446
2447 void
2448 snat_free_outside_address_and_port (snat_address_t * addresses,
2449                                     u32 thread_index, snat_session_key_t * k)
2450 {
2451   snat_address_t *a;
2452   u32 address_index;
2453   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2454
2455   for (address_index = 0; address_index < vec_len (addresses);
2456        address_index++)
2457     {
2458       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2459         break;
2460     }
2461
2462   ASSERT (address_index < vec_len (addresses));
2463
2464   a = addresses + address_index;
2465
2466   switch (k->protocol)
2467     {
2468 #define _(N, i, n, s) \
2469     case SNAT_PROTOCOL_##N: \
2470       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2471         port_host_byte_order) == 1); \
2472       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2473         port_host_byte_order, 0); \
2474       a->busy_##n##_ports--; \
2475       a->busy_##n##_ports_per_thread[thread_index]--; \
2476       break;
2477       foreach_snat_protocol
2478 #undef _
2479     default:
2480       nat_elog_info ("unknown protocol");
2481       return;
2482     }
2483 }
2484
2485 static int
2486 nat_set_outside_address_and_port (snat_address_t * addresses,
2487                                   u32 thread_index, snat_session_key_t * k)
2488 {
2489   snat_address_t *a = 0;
2490   u32 address_index;
2491   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2492
2493   for (address_index = 0; address_index < vec_len (addresses);
2494        address_index++)
2495     {
2496       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2497         continue;
2498
2499       a = addresses + address_index;
2500       switch (k->protocol)
2501         {
2502 #define _(N, j, n, s) \
2503         case SNAT_PROTOCOL_##N: \
2504           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2505             return VNET_API_ERROR_INSTANCE_IN_USE; \
2506           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2507           a->busy_##n##_ports_per_thread[thread_index]++; \
2508           a->busy_##n##_ports++; \
2509           return 0;
2510           foreach_snat_protocol
2511 #undef _
2512         default:
2513           nat_elog_info ("unknown protocol");
2514           return 1;
2515         }
2516     }
2517
2518   return VNET_API_ERROR_NO_SUCH_ENTRY;
2519 }
2520
2521 int
2522 snat_static_mapping_match (snat_main_t * sm,
2523                            snat_session_key_t match,
2524                            snat_session_key_t * mapping,
2525                            u8 by_external,
2526                            u8 * is_addr_only,
2527                            twice_nat_type_t * twice_nat,
2528                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2529                            u8 * is_identity_nat)
2530 {
2531   clib_bihash_kv_8_8_t kv, value;
2532   snat_static_mapping_t *m;
2533   snat_session_key_t m_key;
2534   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2535   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2536   u8 backend_index;
2537   nat44_lb_addr_port_t *local;
2538
2539   m_key.fib_index = match.fib_index;
2540   if (by_external)
2541     {
2542       mapping_hash = &sm->static_mapping_by_external;
2543       m_key.fib_index = 0;
2544     }
2545
2546   m_key.addr = match.addr;
2547   m_key.port = clib_net_to_host_u16 (match.port);
2548   m_key.protocol = match.protocol;
2549
2550   kv.key = m_key.as_u64;
2551
2552   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2553     {
2554       /* Try address only mapping */
2555       m_key.port = 0;
2556       m_key.protocol = 0;
2557       kv.key = m_key.as_u64;
2558       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2559         return 1;
2560     }
2561
2562   m = pool_elt_at_index (sm->static_mappings, value.value);
2563
2564   if (by_external)
2565     {
2566       if (is_lb_static_mapping (m))
2567         {
2568           if (PREDICT_FALSE (lb != 0))
2569             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2570           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2571                                                           match.addr,
2572                                                           match.protocol,
2573                                                           match.port,
2574                                                           &backend_index))
2575             {
2576               local = pool_elt_at_index (m->locals, backend_index);
2577               mapping->addr = local->addr;
2578               mapping->port = clib_host_to_net_u16 (local->port);
2579               mapping->fib_index = local->fib_index;
2580               goto end;
2581             }
2582           // pick locals matching this worker
2583           if (PREDICT_FALSE (sm->num_workers > 1))
2584             {
2585               u32 thread_index = vlib_get_thread_index ();
2586               /* *INDENT-OFF* */
2587               pool_foreach_index (i, m->locals,
2588               ({
2589                 local = pool_elt_at_index (m->locals, i);
2590
2591                 ip4_header_t ip = {
2592                   .src_address = local->addr,
2593                 };
2594
2595                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2596                     thread_index)
2597                   {
2598                     vec_add1 (tmp, i);
2599                   }
2600               }));
2601               /* *INDENT-ON* */
2602               ASSERT (vec_len (tmp) != 0);
2603             }
2604           else
2605             {
2606               /* *INDENT-OFF* */
2607               pool_foreach_index (i, m->locals,
2608               ({
2609                 vec_add1 (tmp, i);
2610               }));
2611               /* *INDENT-ON* */
2612             }
2613           hi = vec_len (tmp) - 1;
2614           local = pool_elt_at_index (m->locals, tmp[hi]);
2615           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2616           while (lo < hi)
2617             {
2618               mid = ((hi - lo) >> 1) + lo;
2619               local = pool_elt_at_index (m->locals, tmp[mid]);
2620               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2621             }
2622           local = pool_elt_at_index (m->locals, tmp[lo]);
2623           if (!(local->prefix >= rand))
2624             return 1;
2625           mapping->addr = local->addr;
2626           mapping->port = clib_host_to_net_u16 (local->port);
2627           mapping->fib_index = local->fib_index;
2628           if (m->affinity)
2629             {
2630               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2631                                                 match.protocol, match.port,
2632                                                 tmp[lo], m->affinity,
2633                                                 m->affinity_per_service_list_head_index))
2634                 nat_elog_info ("create affinity record failed");
2635             }
2636           vec_free (tmp);
2637         }
2638       else
2639         {
2640           if (PREDICT_FALSE (lb != 0))
2641             *lb = NO_LB_NAT;
2642           mapping->fib_index = m->fib_index;
2643           mapping->addr = m->local_addr;
2644           /* Address only mapping doesn't change port */
2645           mapping->port = is_addr_only_static_mapping (m) ? match.port
2646             : clib_host_to_net_u16 (m->local_port);
2647         }
2648       mapping->protocol = m->proto;
2649     }
2650   else
2651     {
2652       mapping->addr = m->external_addr;
2653       /* Address only mapping doesn't change port */
2654       mapping->port = is_addr_only_static_mapping (m) ? match.port
2655         : clib_host_to_net_u16 (m->external_port);
2656       mapping->fib_index = sm->outside_fib_index;
2657     }
2658
2659 end:
2660   if (PREDICT_FALSE (is_addr_only != 0))
2661     *is_addr_only = is_addr_only_static_mapping (m);
2662
2663   if (PREDICT_FALSE (twice_nat != 0))
2664     *twice_nat = m->twice_nat;
2665
2666   if (PREDICT_FALSE (is_identity_nat != 0))
2667     *is_identity_nat = is_identity_static_mapping (m);
2668
2669   return 0;
2670 }
2671
2672 static_always_inline u16
2673 snat_random_port (u16 min, u16 max)
2674 {
2675   snat_main_t *sm = &snat_main;
2676   return min + random_u32 (&sm->random_seed) /
2677     (random_u32_max () / (max - min + 1) + 1);
2678 }
2679
2680 int
2681 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2682                                      u32 fib_index,
2683                                      u32 thread_index,
2684                                      snat_session_key_t * k,
2685                                      u16 port_per_thread,
2686                                      u32 snat_thread_index)
2687 {
2688   snat_main_t *sm = &snat_main;
2689
2690   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2691                                   port_per_thread, snat_thread_index);
2692 }
2693
2694 static int
2695 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2696                                  u32 fib_index,
2697                                  u32 thread_index,
2698                                  snat_session_key_t * k,
2699                                  u16 port_per_thread, u32 snat_thread_index)
2700 {
2701   int i;
2702   snat_address_t *a, *ga = 0;
2703   u32 portnum;
2704
2705   for (i = 0; i < vec_len (addresses); i++)
2706     {
2707       a = addresses + i;
2708       switch (k->protocol)
2709         {
2710 #define _(N, j, n, s) \
2711         case SNAT_PROTOCOL_##N: \
2712           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2713             { \
2714               if (a->fib_index == fib_index) \
2715                 { \
2716                   while (1) \
2717                     { \
2718                       portnum = (port_per_thread * \
2719                         snat_thread_index) + \
2720                         snat_random_port(1, port_per_thread) + 1024; \
2721                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2722                         continue; \
2723                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2724                       a->busy_##n##_ports_per_thread[thread_index]++; \
2725                       a->busy_##n##_ports++; \
2726                       k->addr = a->addr; \
2727                       k->port = clib_host_to_net_u16(portnum); \
2728                       return 0; \
2729                     } \
2730                 } \
2731               else if (a->fib_index == ~0) \
2732                 { \
2733                   ga = a; \
2734                 } \
2735             } \
2736           break;
2737           foreach_snat_protocol
2738 #undef _
2739         default:
2740           nat_elog_info ("unknown protocol");
2741           return 1;
2742         }
2743
2744     }
2745
2746   if (ga)
2747     {
2748       a = ga;
2749       switch (k->protocol)
2750         {
2751 #define _(N, j, n, s) \
2752         case SNAT_PROTOCOL_##N: \
2753           while (1) \
2754             { \
2755               portnum = (port_per_thread * \
2756                 snat_thread_index) + \
2757                 snat_random_port(1, port_per_thread) + 1024; \
2758               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2759                 continue; \
2760               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2761               a->busy_##n##_ports_per_thread[thread_index]++; \
2762               a->busy_##n##_ports++; \
2763               k->addr = a->addr; \
2764               k->port = clib_host_to_net_u16(portnum); \
2765               return 0; \
2766             }
2767           break;
2768           foreach_snat_protocol
2769 #undef _
2770         default:
2771           nat_elog_info ("unknown protocol");
2772           return 1;
2773         }
2774     }
2775
2776   /* Totally out of translations to use... */
2777   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2778   return 1;
2779 }
2780
2781 static int
2782 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2783                               u32 fib_index,
2784                               u32 thread_index,
2785                               snat_session_key_t * k,
2786                               u16 port_per_thread, u32 snat_thread_index)
2787 {
2788   snat_main_t *sm = &snat_main;
2789   snat_address_t *a = addresses;
2790   u16 m, ports, portnum, A, j;
2791   m = 16 - (sm->psid_offset + sm->psid_length);
2792   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2793
2794   if (!vec_len (addresses))
2795     goto exhausted;
2796
2797   switch (k->protocol)
2798     {
2799 #define _(N, i, n, s) \
2800     case SNAT_PROTOCOL_##N: \
2801       if (a->busy_##n##_ports < ports) \
2802         { \
2803           while (1) \
2804             { \
2805               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2806               j = snat_random_port(0, pow2_mask(m)); \
2807               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2808               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2809                 continue; \
2810               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2811               a->busy_##n##_ports++; \
2812               k->addr = a->addr; \
2813               k->port = clib_host_to_net_u16 (portnum); \
2814               return 0; \
2815             } \
2816         } \
2817       break;
2818       foreach_snat_protocol
2819 #undef _
2820     default:
2821       nat_elog_info ("unknown protocol");
2822       return 1;
2823     }
2824
2825 exhausted:
2826   /* Totally out of translations to use... */
2827   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2828   return 1;
2829 }
2830
2831 static int
2832 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2833                                u32 fib_index,
2834                                u32 thread_index,
2835                                snat_session_key_t * k,
2836                                u16 port_per_thread, u32 snat_thread_index)
2837 {
2838   snat_main_t *sm = &snat_main;
2839   snat_address_t *a = addresses;
2840   u16 portnum, ports;
2841
2842   ports = sm->end_port - sm->start_port + 1;
2843
2844   if (!vec_len (addresses))
2845     goto exhausted;
2846
2847   switch (k->protocol)
2848     {
2849 #define _(N, i, n, s) \
2850     case SNAT_PROTOCOL_##N: \
2851       if (a->busy_##n##_ports < ports) \
2852         { \
2853           while (1) \
2854             { \
2855               portnum = snat_random_port(sm->start_port, sm->end_port); \
2856               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2857                 continue; \
2858               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2859               a->busy_##n##_ports++; \
2860               k->addr = a->addr; \
2861               k->port = clib_host_to_net_u16 (portnum); \
2862               return 0; \
2863             } \
2864         } \
2865       break;
2866       foreach_snat_protocol
2867 #undef _
2868     default:
2869       nat_elog_info ("unknown protocol");
2870       return 1;
2871     }
2872
2873 exhausted:
2874   /* Totally out of translations to use... */
2875   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2876   return 1;
2877 }
2878
2879 void
2880 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2881 {
2882   dpo_id_t dpo_v4 = DPO_INVALID;
2883   fib_prefix_t pfx = {
2884     .fp_proto = FIB_PROTOCOL_IP4,
2885     .fp_len = 32,
2886     .fp_addr.ip4.as_u32 = addr.as_u32,
2887   };
2888
2889   if (is_add)
2890     {
2891       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2892       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2893                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2894       dpo_reset (&dpo_v4);
2895     }
2896   else
2897     {
2898       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2899     }
2900 }
2901
2902 u8 *
2903 format_session_kvp (u8 * s, va_list * args)
2904 {
2905   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2906   snat_session_key_t k;
2907
2908   k.as_u64 = v->key;
2909
2910   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2911
2912   return s;
2913 }
2914
2915 u8 *
2916 format_static_mapping_kvp (u8 * s, va_list * args)
2917 {
2918   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2919   snat_session_key_t k;
2920
2921   k.as_u64 = v->key;
2922
2923   s = format (s, "%U static-mapping-index %llu",
2924               format_static_mapping_key, &k, v->value);
2925
2926   return s;
2927 }
2928
2929 u8 *
2930 format_user_kvp (u8 * s, va_list * args)
2931 {
2932   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2933   snat_user_key_t k;
2934
2935   k.as_u64 = v->key;
2936
2937   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2938               k.fib_index, v->value);
2939
2940   return s;
2941 }
2942
2943 u8 *
2944 format_ed_session_kvp (u8 * s, va_list * args)
2945 {
2946   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2947   nat_ed_ses_key_t k;
2948
2949   k.as_u64[0] = v->key[0];
2950   k.as_u64[1] = v->key[1];
2951
2952   s =
2953     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2954             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2955             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2956             format_ip_protocol, k.proto, k.fib_index, v->value);
2957
2958   return s;
2959 }
2960
2961 static u32
2962 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
2963                            u8 is_output)
2964 {
2965   snat_main_t *sm = &snat_main;
2966   u32 next_worker_index = 0;
2967   u32 hash;
2968
2969   next_worker_index = sm->first_worker_index;
2970   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2971     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2972
2973   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2974     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2975   else
2976     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2977
2978   return next_worker_index;
2979 }
2980
2981 static u32
2982 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0,
2983                            u8 is_output)
2984 {
2985   snat_main_t *sm = &snat_main;
2986   udp_header_t *udp;
2987   u16 port;
2988   snat_session_key_t m_key;
2989   clib_bihash_kv_8_8_t kv, value;
2990   snat_static_mapping_t *m;
2991   u32 proto;
2992   u32 next_worker_index = 0;
2993
2994   /* first try static mappings without port */
2995   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2996     {
2997       m_key.addr = ip0->dst_address;
2998       m_key.port = 0;
2999       m_key.protocol = 0;
3000       m_key.fib_index = rx_fib_index0;
3001       kv.key = m_key.as_u64;
3002       if (!clib_bihash_search_8_8
3003           (&sm->static_mapping_by_external, &kv, &value))
3004         {
3005           m = pool_elt_at_index (sm->static_mappings, value.value);
3006           return m->workers[0];
3007         }
3008     }
3009
3010   proto = ip_proto_to_snat_proto (ip0->protocol);
3011   udp = ip4_next_header (ip0);
3012   port = udp->dst_port;
3013
3014   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
3015     {
3016       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
3017         return vlib_get_thread_index ();
3018
3019       nat_reass_ip4_t *reass;
3020       reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
3021                                   ip0->fragment_id, ip0->protocol);
3022
3023       if (reass && (reass->thread_index != (u32) ~ 0))
3024         return reass->thread_index;
3025
3026       if (ip4_is_first_fragment (ip0))
3027         {
3028           reass =
3029             nat_ip4_reass_create (ip0->src_address, ip0->dst_address,
3030                                   ip0->fragment_id, ip0->protocol);
3031           if (!reass)
3032             goto no_reass;
3033
3034           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3035             {
3036               m_key.addr = ip0->dst_address;
3037               m_key.port = clib_net_to_host_u16 (port);
3038               m_key.protocol = proto;
3039               m_key.fib_index = rx_fib_index0;
3040               kv.key = m_key.as_u64;
3041               if (!clib_bihash_search_8_8
3042                   (&sm->static_mapping_by_external, &kv, &value))
3043                 {
3044                   m = pool_elt_at_index (sm->static_mappings, value.value);
3045                   reass->thread_index = m->workers[0];
3046                   return reass->thread_index;
3047                 }
3048             }
3049           reass->thread_index = sm->first_worker_index;
3050           reass->thread_index +=
3051             sm->workers[(clib_net_to_host_u16 (port) - 1024) /
3052                         sm->port_per_thread];
3053           return reass->thread_index;
3054         }
3055       else
3056         return vlib_get_thread_index ();
3057     }
3058
3059 no_reass:
3060   /* unknown protocol */
3061   if (PREDICT_FALSE (proto == ~0))
3062     {
3063       /* use current thread */
3064       return vlib_get_thread_index ();
3065     }
3066
3067   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3068     {
3069       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3070       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3071       if (!icmp_is_error_message (icmp))
3072         port = echo->identifier;
3073       else
3074         {
3075           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3076           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3077           void *l4_header = ip4_next_header (inner_ip);
3078           switch (proto)
3079             {
3080             case SNAT_PROTOCOL_ICMP:
3081               icmp = (icmp46_header_t *) l4_header;
3082               echo = (icmp_echo_header_t *) (icmp + 1);
3083               port = echo->identifier;
3084               break;
3085             case SNAT_PROTOCOL_UDP:
3086             case SNAT_PROTOCOL_TCP:
3087               port = ((tcp_udp_header_t *) l4_header)->src_port;
3088               break;
3089             default:
3090               return vlib_get_thread_index ();
3091             }
3092         }
3093     }
3094
3095   /* try static mappings with port */
3096   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3097     {
3098       m_key.addr = ip0->dst_address;
3099       m_key.port = clib_net_to_host_u16 (port);
3100       m_key.protocol = proto;
3101       m_key.fib_index = rx_fib_index0;
3102       kv.key = m_key.as_u64;
3103       if (!clib_bihash_search_8_8
3104           (&sm->static_mapping_by_external, &kv, &value))
3105         {
3106           m = pool_elt_at_index (sm->static_mappings, value.value);
3107           return m->workers[0];
3108         }
3109     }
3110
3111   /* worker by outside port */
3112   next_worker_index = sm->first_worker_index;
3113   next_worker_index +=
3114     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3115   return next_worker_index;
3116 }
3117
3118 static u32
3119 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3120                                u8 is_output)
3121 {
3122   snat_main_t *sm = &snat_main;
3123   u32 next_worker_index = sm->first_worker_index;
3124   u32 hash;
3125
3126   clib_bihash_kv_16_8_t kv16, value16;
3127   snat_main_per_thread_data_t *tsm;
3128   udp_header_t *udp;
3129
3130   if (PREDICT_FALSE (is_output))
3131     {
3132       u32 fib_index = sm->outside_fib_index;
3133       nat_outside_fib_t *outside_fib;
3134       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3135       fib_prefix_t pfx = {
3136         .fp_proto = FIB_PROTOCOL_IP4,
3137         .fp_len = 32,
3138         .fp_addr = {
3139                     .ip4.as_u32 = ip->dst_address.as_u32,
3140                     }
3141         ,
3142       };
3143
3144       udp = ip4_next_header (ip);
3145
3146       switch (vec_len (sm->outside_fibs))
3147         {
3148         case 0:
3149           fib_index = sm->outside_fib_index;
3150           break;
3151         case 1:
3152           fib_index = sm->outside_fibs[0].fib_index;
3153           break;
3154         default:
3155             /* *INDENT-OFF* */
3156             vec_foreach (outside_fib, sm->outside_fibs)
3157               {
3158                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3159                 if (FIB_NODE_INDEX_INVALID != fei)
3160                   {
3161                     if (fib_entry_get_resolving_interface (fei) != ~0)
3162                       {
3163                         fib_index = outside_fib->fib_index;
3164                         break;
3165                       }
3166                   }
3167               }
3168             /* *INDENT-ON* */
3169           break;
3170         }
3171
3172       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3173                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3174
3175       /* *INDENT-OFF* */
3176       vec_foreach (tsm, sm->per_thread_data)
3177         {
3178           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3179                                                       &kv16, &value16)))
3180             {
3181               next_worker_index += tsm->thread_index;
3182
3183               nat_elog_debug_handoff (
3184                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3185                 next_worker_index, fib_index,
3186                 clib_net_to_host_u32 (ip->src_address.as_u32),
3187                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3188
3189               return next_worker_index;
3190             }
3191         }
3192       /* *INDENT-ON* */
3193     }
3194
3195   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3196     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3197
3198   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3199     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3200   else
3201     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3202
3203   if (PREDICT_TRUE (!is_output))
3204     {
3205       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3206                               next_worker_index, rx_fib_index,
3207                               clib_net_to_host_u32 (ip->src_address.as_u32),
3208                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3209     }
3210   else
3211     {
3212       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3213                               next_worker_index, rx_fib_index,
3214                               clib_net_to_host_u32 (ip->src_address.as_u32),
3215                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3216     }
3217
3218   return next_worker_index;
3219 }
3220
3221 static u32
3222 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index,
3223                                u8 is_output)
3224 {
3225   snat_main_t *sm = &snat_main;
3226   clib_bihash_kv_8_8_t kv, value;
3227   clib_bihash_kv_16_8_t kv16, value16;
3228   snat_main_per_thread_data_t *tsm;
3229
3230   u32 proto, next_worker_index = 0;
3231   udp_header_t *udp;
3232   u16 port;
3233   snat_static_mapping_t *m;
3234   u32 hash;
3235
3236   proto = ip_proto_to_snat_proto (ip->protocol);
3237
3238   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3239     {
3240       udp = ip4_next_header (ip);
3241
3242       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3243                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3244
3245       /* *INDENT-OFF* */
3246       vec_foreach (tsm, sm->per_thread_data)
3247         {
3248           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3249                                                       &kv16, &value16)))
3250             {
3251               next_worker_index = sm->first_worker_index + tsm->thread_index;
3252               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3253                           next_worker_index, rx_fib_index,
3254                           clib_net_to_host_u32 (ip->src_address.as_u32),
3255                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3256               return next_worker_index;
3257             }
3258           }
3259         /* *INDENT-ON* */
3260     }
3261   else if (proto == SNAT_PROTOCOL_ICMP)
3262     {
3263       nat_ed_ses_key_t key;
3264
3265       if (!get_icmp_o2i_ed_key (ip, &key))
3266         {
3267
3268           key.fib_index = rx_fib_index;
3269           kv16.key[0] = key.as_u64[0];
3270           kv16.key[1] = key.as_u64[1];
3271
3272           /* *INDENT-OFF* */
3273           vec_foreach (tsm, sm->per_thread_data)
3274             {
3275               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3276                                                           &kv16, &value16)))
3277                 {
3278                   next_worker_index = sm->first_worker_index +
3279                                       tsm->thread_index;
3280                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3281                               next_worker_index, rx_fib_index,
3282                               clib_net_to_host_u32 (ip->src_address.as_u32),
3283                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3284                   return next_worker_index;
3285                 }
3286             }
3287           /* *INDENT-ON* */
3288         }
3289     }
3290
3291   /* first try static mappings without port */
3292   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3293     {
3294       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3295       if (!clib_bihash_search_8_8
3296           (&sm->static_mapping_by_external, &kv, &value))
3297         {
3298           m = pool_elt_at_index (sm->static_mappings, value.value);
3299           next_worker_index = m->workers[0];
3300           goto done;
3301         }
3302     }
3303
3304   /* unknown protocol */
3305   if (PREDICT_FALSE (proto == ~0))
3306     {
3307       /* use current thread */
3308       next_worker_index = vlib_get_thread_index ();
3309       goto done;
3310     }
3311
3312   udp = ip4_next_header (ip);
3313   port = udp->dst_port;
3314
3315   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3316     {
3317       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3318       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3319       if (!icmp_is_error_message (icmp))
3320         port = echo->identifier;
3321       else
3322         {
3323           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3324           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3325           void *l4_header = ip4_next_header (inner_ip);
3326           switch (proto)
3327             {
3328             case SNAT_PROTOCOL_ICMP:
3329               icmp = (icmp46_header_t *) l4_header;
3330               echo = (icmp_echo_header_t *) (icmp + 1);
3331               port = echo->identifier;
3332               break;
3333             case SNAT_PROTOCOL_UDP:
3334             case SNAT_PROTOCOL_TCP:
3335               port = ((tcp_udp_header_t *) l4_header)->src_port;
3336               break;
3337             default:
3338               next_worker_index = vlib_get_thread_index ();
3339               goto done;
3340             }
3341         }
3342     }
3343
3344   /* try static mappings with port */
3345   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3346     {
3347       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3348                   clib_net_to_host_u16 (port));
3349       if (!clib_bihash_search_8_8
3350           (&sm->static_mapping_by_external, &kv, &value))
3351         {
3352           m = pool_elt_at_index (sm->static_mappings, value.value);
3353           if (!is_lb_static_mapping (m))
3354             {
3355               next_worker_index = m->workers[0];
3356               goto done;
3357             }
3358
3359           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3360             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3361
3362           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3363             next_worker_index =
3364               m->workers[hash & (_vec_len (m->workers) - 1)];
3365           else
3366             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3367           goto done;
3368         }
3369     }
3370
3371   /* worker by outside port */
3372   next_worker_index = sm->first_worker_index;
3373   next_worker_index +=
3374     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3375
3376 done:
3377   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3378                           clib_net_to_host_u32 (ip->src_address.as_u32),
3379                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3380   return next_worker_index;
3381 }
3382
3383 void
3384 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3385                 ip4_address_t * out_addr, u16 out_port,
3386                 ip4_address_t * eh_addr, u16 eh_port,
3387                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3388                 u32 fib_index, u16 flags, u32 thread_index)
3389 {
3390   snat_main_t *sm = &snat_main;
3391   snat_session_key_t key;
3392   snat_user_t *u;
3393   snat_session_t *s;
3394   clib_bihash_kv_8_8_t kv;
3395   f64 now = vlib_time_now (sm->vlib_main);
3396   nat_outside_fib_t *outside_fib;
3397   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3398   snat_main_per_thread_data_t *tsm;
3399   fib_prefix_t pfx = {
3400     .fp_proto = FIB_PROTOCOL_IP4,
3401     .fp_len = 32,
3402     .fp_addr = {
3403                 .ip4.as_u32 = eh_addr->as_u32,
3404                 },
3405   };
3406
3407   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3408
3409   key.addr.as_u32 = out_addr->as_u32;
3410   key.port = out_port;
3411   key.protocol = proto;
3412
3413   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3414     {
3415       if (nat_set_outside_address_and_port
3416           (sm->addresses, thread_index, &key))
3417         return;
3418     }
3419
3420   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3421   if (!u)
3422     return;
3423
3424   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3425   if (!s)
3426     return;
3427
3428   s->last_heard = now;
3429   s->flags = flags;
3430   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3431   s->ext_host_port = eh_port;
3432   user_session_increment (sm, u, snat_is_session_static (s));
3433   switch (vec_len (sm->outside_fibs))
3434     {
3435     case 0:
3436       key.fib_index = sm->outside_fib_index;
3437       break;
3438     case 1:
3439       key.fib_index = sm->outside_fibs[0].fib_index;
3440       break;
3441     default:
3442       /* *INDENT-OFF* */
3443       vec_foreach (outside_fib, sm->outside_fibs)
3444         {
3445           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3446           if (FIB_NODE_INDEX_INVALID != fei)
3447             {
3448               if (fib_entry_get_resolving_interface (fei) != ~0)
3449                 {
3450                   key.fib_index = outside_fib->fib_index;
3451                   break;
3452                 }
3453             }
3454         }
3455       /* *INDENT-ON* */
3456       break;
3457     }
3458   s->out2in = key;
3459   kv.key = key.as_u64;
3460   kv.value = s - tsm->sessions;
3461   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3462     nat_elog_warn ("out2in key add failed");
3463
3464   key.addr.as_u32 = in_addr->as_u32;
3465   key.port = in_port;
3466   key.fib_index = fib_index;
3467   s->in2out = key;
3468   kv.key = key.as_u64;
3469   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3470     nat_elog_warn ("in2out key add failed");
3471 }
3472
3473 void
3474 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3475                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3476                 u32 ti)
3477 {
3478   snat_main_t *sm = &snat_main;
3479   snat_session_key_t key;
3480   clib_bihash_kv_8_8_t kv, value;
3481   u32 thread_index;
3482   snat_session_t *s;
3483   snat_main_per_thread_data_t *tsm;
3484
3485   if (sm->num_workers > 1)
3486     thread_index =
3487       sm->first_worker_index +
3488       (sm->workers[(clib_net_to_host_u16 (out_port) -
3489                     1024) / sm->port_per_thread]);
3490   else
3491     thread_index = sm->num_workers;
3492   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3493
3494   key.addr.as_u32 = out_addr->as_u32;
3495   key.port = out_port;
3496   key.protocol = proto;
3497   key.fib_index = fib_index;
3498   kv.key = key.as_u64;
3499   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3500     return;
3501
3502   s = pool_elt_at_index (tsm->sessions, value.value);
3503   nat_free_session_data (sm, s, thread_index, 1);
3504   nat44_delete_session (sm, s, thread_index);
3505 }
3506
3507 void
3508 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3509                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3510                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3511 {
3512   snat_main_t *sm = &snat_main;
3513   snat_session_key_t key;
3514   clib_bihash_kv_8_8_t kv, value;
3515   snat_session_t *s;
3516   snat_main_per_thread_data_t *tsm;
3517
3518   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3519
3520   key.addr.as_u32 = out_addr->as_u32;
3521   key.port = out_port;
3522   key.protocol = proto;
3523   key.fib_index = fib_index;
3524   kv.key = key.as_u64;
3525   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3526     return;
3527
3528   s = pool_elt_at_index (tsm->sessions, value.value);
3529   s->total_pkts = total_pkts;
3530   s->total_bytes = total_bytes;
3531 }
3532
3533 void
3534 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3535                    ip4_address_t * out_addr, u16 out_port,
3536                    ip4_address_t * eh_addr, u16 eh_port,
3537                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3538                    u32 fib_index, u16 flags, u32 thread_index)
3539 {
3540   snat_main_t *sm = &snat_main;
3541   snat_session_key_t key;
3542   snat_user_t *u;
3543   snat_session_t *s;
3544   clib_bihash_kv_16_8_t kv;
3545   f64 now = vlib_time_now (sm->vlib_main);
3546   nat_outside_fib_t *outside_fib;
3547   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3548   snat_main_per_thread_data_t *tsm;
3549   fib_prefix_t pfx = {
3550     .fp_proto = FIB_PROTOCOL_IP4,
3551     .fp_len = 32,
3552     .fp_addr = {
3553                 .ip4.as_u32 = eh_addr->as_u32,
3554                 },
3555   };
3556
3557   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3558
3559   key.addr.as_u32 = out_addr->as_u32;
3560   key.port = out_port;
3561   key.protocol = proto;
3562
3563   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3564     {
3565       if (nat_set_outside_address_and_port
3566           (sm->addresses, thread_index, &key))
3567         return;
3568     }
3569
3570   key.addr.as_u32 = ehn_addr->as_u32;
3571   key.port = ehn_port;
3572   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3573     {
3574       if (nat_set_outside_address_and_port
3575           (sm->twice_nat_addresses, thread_index, &key))
3576         return;
3577     }
3578
3579   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3580   if (!u)
3581     return;
3582
3583   s = nat_ed_session_alloc (sm, u, thread_index, now);
3584   if (!s)
3585     return;
3586
3587   s->last_heard = now;
3588   s->flags = flags;
3589   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3590   s->ext_host_nat_port = s->ext_host_port = eh_port;
3591   if (is_twice_nat_session (s))
3592     {
3593       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3594       s->ext_host_nat_port = ehn_port;
3595     }
3596   user_session_increment (sm, u, snat_is_session_static (s));
3597   switch (vec_len (sm->outside_fibs))
3598     {
3599     case 0:
3600       key.fib_index = sm->outside_fib_index;
3601       break;
3602     case 1:
3603       key.fib_index = sm->outside_fibs[0].fib_index;
3604       break;
3605     default:
3606       /* *INDENT-OFF* */
3607       vec_foreach (outside_fib, sm->outside_fibs)
3608         {
3609           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3610           if (FIB_NODE_INDEX_INVALID != fei)
3611             {
3612               if (fib_entry_get_resolving_interface (fei) != ~0)
3613                 {
3614                   key.fib_index = outside_fib->fib_index;
3615                   break;
3616                 }
3617             }
3618         }
3619       /* *INDENT-ON* */
3620       break;
3621     }
3622   key.addr.as_u32 = out_addr->as_u32;
3623   key.port = out_port;
3624   s->out2in = key;
3625   kv.value = s - tsm->sessions;
3626
3627   key.addr.as_u32 = in_addr->as_u32;
3628   key.port = in_port;
3629   key.fib_index = fib_index;
3630   s->in2out = key;
3631
3632   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3633               snat_proto_to_ip_proto (proto), fib_index, in_port,
3634               s->ext_host_nat_port);
3635   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3636     nat_elog_warn ("in2out key add failed");
3637
3638   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3639               s->out2in.fib_index, out_port, eh_port);
3640   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3641     nat_elog_warn ("out2in key add failed");
3642 }
3643
3644 void
3645 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3646                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3647                    u32 fib_index, u32 ti)
3648 {
3649   snat_main_t *sm = &snat_main;
3650   nat_ed_ses_key_t key;
3651   clib_bihash_kv_16_8_t kv, value;
3652   u32 thread_index;
3653   snat_session_t *s;
3654   snat_main_per_thread_data_t *tsm;
3655
3656   if (sm->num_workers > 1)
3657     thread_index =
3658       sm->first_worker_index +
3659       (sm->workers[(clib_net_to_host_u16 (out_port) -
3660                     1024) / sm->port_per_thread]);
3661   else
3662     thread_index = sm->num_workers;
3663   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3664
3665   key.l_addr.as_u32 = out_addr->as_u32;
3666   key.l_port = out_port;
3667   key.r_addr.as_u32 = eh_addr->as_u32;
3668   key.r_port = eh_port;
3669   key.proto = proto;
3670   key.fib_index = fib_index;
3671   kv.key[0] = key.as_u64[0];
3672   kv.key[1] = key.as_u64[1];
3673   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3674     return;
3675
3676   s = pool_elt_at_index (tsm->sessions, value.value);
3677   nat_free_session_data (sm, s, thread_index, 1);
3678   nat44_delete_session (sm, s, thread_index);
3679 }
3680
3681 void
3682 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3683                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3684                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3685                    u32 thread_index)
3686 {
3687   snat_main_t *sm = &snat_main;
3688   nat_ed_ses_key_t key;
3689   clib_bihash_kv_16_8_t kv, value;
3690   snat_session_t *s;
3691   snat_main_per_thread_data_t *tsm;
3692
3693   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3694
3695   key.l_addr.as_u32 = out_addr->as_u32;
3696   key.l_port = out_port;
3697   key.r_addr.as_u32 = eh_addr->as_u32;
3698   key.r_port = eh_port;
3699   key.proto = proto;
3700   key.fib_index = fib_index;
3701   kv.key[0] = key.as_u64[0];
3702   kv.key[1] = key.as_u64[1];
3703   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3704     return;
3705
3706   s = pool_elt_at_index (tsm->sessions, value.value);
3707   s->total_pkts = total_pkts;
3708   s->total_bytes = total_bytes;
3709 }
3710
3711 static clib_error_t *
3712 snat_config (vlib_main_t * vm, unformat_input_t * input)
3713 {
3714   snat_main_t *sm = &snat_main;
3715   nat66_main_t *nm = &nat66_main;
3716   u32 translation_buckets = 1024;
3717   u32 translation_memory_size = 128 << 20;
3718   u32 user_buckets = 128;
3719   u32 user_memory_size = 64 << 20;
3720   u32 max_translations_per_user = 100;
3721   u32 outside_vrf_id = 0;
3722   u32 outside_ip6_vrf_id = 0;
3723   u32 inside_vrf_id = 0;
3724   u32 static_mapping_buckets = 1024;
3725   u32 static_mapping_memory_size = 64 << 20;
3726   u32 nat64_bib_buckets = 1024;
3727   u32 nat64_bib_memory_size = 128 << 20;
3728   u32 nat64_st_buckets = 2048;
3729   u32 nat64_st_memory_size = 256 << 20;
3730   u8 static_mapping_only = 0;
3731   u8 static_mapping_connection_tracking = 0;
3732   snat_main_per_thread_data_t *tsm;
3733   dslite_main_t *dm = &dslite_main;
3734
3735   sm->deterministic = 0;
3736   sm->out2in_dpo = 0;
3737   sm->endpoint_dependent = 0;
3738
3739   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3740     {
3741       if (unformat
3742           (input, "translation hash buckets %d", &translation_buckets))
3743         ;
3744       else if (unformat (input, "translation hash memory %d",
3745                          &translation_memory_size));
3746       else if (unformat (input, "user hash buckets %d", &user_buckets))
3747         ;
3748       else if (unformat (input, "user hash memory %d", &user_memory_size))
3749         ;
3750       else if (unformat (input, "max translations per user %d",
3751                          &max_translations_per_user))
3752         ;
3753       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3754         ;
3755       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3756         ;
3757       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3758         ;
3759       else if (unformat (input, "static mapping only"))
3760         {
3761           static_mapping_only = 1;
3762           if (unformat (input, "connection tracking"))
3763             static_mapping_connection_tracking = 1;
3764         }
3765       else if (unformat (input, "deterministic"))
3766         sm->deterministic = 1;
3767       else if (unformat (input, "nat64 bib hash buckets %d",
3768                          &nat64_bib_buckets))
3769         ;
3770       else if (unformat (input, "nat64 bib hash memory %d",
3771                          &nat64_bib_memory_size))
3772         ;
3773       else
3774         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3775         ;
3776       else if (unformat (input, "nat64 st hash memory %d",
3777                          &nat64_st_memory_size))
3778         ;
3779       else if (unformat (input, "out2in dpo"))
3780         sm->out2in_dpo = 1;
3781       else if (unformat (input, "dslite ce"))
3782         dslite_set_ce (dm, 1);
3783       else if (unformat (input, "endpoint-dependent"))
3784         sm->endpoint_dependent = 1;
3785       else
3786         return clib_error_return (0, "unknown input '%U'",
3787                                   format_unformat_error, input);
3788     }
3789
3790   if (sm->deterministic && sm->endpoint_dependent)
3791     return clib_error_return (0,
3792                               "deterministic and endpoint-dependent modes are mutually exclusive");
3793
3794   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3795     return clib_error_return (0,
3796                               "static mapping only mode available only for simple nat");
3797
3798   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3799     return clib_error_return (0,
3800                               "out2in dpo mode available only for simple nat");
3801
3802   /* for show commands, etc. */
3803   sm->translation_buckets = translation_buckets;
3804   sm->translation_memory_size = translation_memory_size;
3805   /* do not exceed load factor 10 */
3806   sm->max_translations = 10 * translation_buckets;
3807   sm->user_buckets = user_buckets;
3808   sm->user_memory_size = user_memory_size;
3809   sm->max_translations_per_user = max_translations_per_user;
3810   sm->outside_vrf_id = outside_vrf_id;
3811   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3812                                                              outside_vrf_id,
3813                                                              FIB_SOURCE_PLUGIN_HI);
3814   nm->outside_vrf_id = outside_ip6_vrf_id;
3815   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3816                                                              outside_ip6_vrf_id,
3817                                                              FIB_SOURCE_PLUGIN_HI);
3818   sm->inside_vrf_id = inside_vrf_id;
3819   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3820                                                             inside_vrf_id,
3821                                                             FIB_SOURCE_PLUGIN_HI);
3822   sm->static_mapping_only = static_mapping_only;
3823   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3824
3825   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3826                   nat64_st_memory_size);
3827
3828   if (sm->deterministic)
3829     {
3830       sm->in2out_node_index = snat_det_in2out_node.index;
3831       sm->in2out_output_node_index = ~0;
3832       sm->out2in_node_index = snat_det_out2in_node.index;
3833       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3834       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3835     }
3836   else
3837     {
3838       if (sm->endpoint_dependent)
3839         {
3840           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
3841           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3842           sm->in2out_node_index = nat44_ed_in2out_node.index;
3843           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3844           sm->out2in_node_index = nat44_ed_out2in_node.index;
3845           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3846           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3847           nat_affinity_init (vm);
3848           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
3849                        nat_ha_sref_ed_cb);
3850         }
3851       else
3852         {
3853           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3854           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3855           sm->in2out_node_index = snat_in2out_node.index;
3856           sm->in2out_output_node_index = snat_in2out_output_node.index;
3857           sm->out2in_node_index = snat_out2in_node.index;
3858           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3859           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3860           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
3861         }
3862       if (!static_mapping_only ||
3863           (static_mapping_only && static_mapping_connection_tracking))
3864         {
3865           /* *INDENT-OFF* */
3866           vec_foreach (tsm, sm->per_thread_data)
3867             {
3868               if (sm->endpoint_dependent)
3869                 {
3870                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3871                                          translation_buckets,
3872                                          translation_memory_size);
3873                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3874                                                       format_ed_session_kvp);
3875
3876                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3877                                          translation_buckets,
3878                                          translation_memory_size);
3879                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3880                                                       format_ed_session_kvp);
3881                 }
3882               else
3883                 {
3884                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3885                                         translation_buckets,
3886                                         translation_memory_size);
3887                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3888                                                      format_session_kvp);
3889
3890                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3891                                         translation_buckets,
3892                                         translation_memory_size);
3893                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3894                                                      format_session_kvp);
3895                 }
3896
3897               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3898                                     user_memory_size);
3899               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3900                                                  format_user_kvp);
3901             }
3902           /* *INDENT-ON* */
3903
3904         }
3905       else
3906         {
3907           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3908           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3909         }
3910       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3911                             "static_mapping_by_local", static_mapping_buckets,
3912                             static_mapping_memory_size);
3913       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3914                                          format_static_mapping_kvp);
3915
3916       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3917                             "static_mapping_by_external",
3918                             static_mapping_buckets,
3919                             static_mapping_memory_size);
3920       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3921                                          format_static_mapping_kvp);
3922     }
3923
3924   return 0;
3925 }
3926
3927 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3928
3929 static void
3930 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3931                                  uword opaque,
3932                                  u32 sw_if_index,
3933                                  ip4_address_t * address,
3934                                  u32 address_length,
3935                                  u32 if_address_index, u32 is_delete)
3936 {
3937   snat_main_t *sm = &snat_main;
3938   snat_static_map_resolve_t *rp;
3939   snat_static_mapping_t *m;
3940   snat_session_key_t m_key;
3941   clib_bihash_kv_8_8_t kv, value;
3942   int i, rv;
3943   ip4_address_t l_addr;
3944
3945   for (i = 0; i < vec_len (sm->to_resolve); i++)
3946     {
3947       rp = sm->to_resolve + i;
3948       if (rp->addr_only == 0)
3949         continue;
3950       if (rp->sw_if_index == sw_if_index)
3951         goto match;
3952     }
3953
3954   return;
3955
3956 match:
3957   m_key.addr.as_u32 = address->as_u32;
3958   m_key.port = rp->addr_only ? 0 : rp->e_port;
3959   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3960   m_key.fib_index = sm->outside_fib_index;
3961   kv.key = m_key.as_u64;
3962   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3963     m = 0;
3964   else
3965     m = pool_elt_at_index (sm->static_mappings, value.value);
3966
3967   if (!is_delete)
3968     {
3969       /* Don't trip over lease renewal, static config */
3970       if (m)
3971         return;
3972     }
3973   else
3974     {
3975       if (!m)
3976         return;
3977     }
3978
3979   /* Indetity mapping? */
3980   if (rp->l_addr.as_u32 == 0)
3981     l_addr.as_u32 = address[0].as_u32;
3982   else
3983     l_addr.as_u32 = rp->l_addr.as_u32;
3984   /* Add the static mapping */
3985   rv = snat_add_static_mapping (l_addr,
3986                                 address[0],
3987                                 rp->l_port,
3988                                 rp->e_port,
3989                                 rp->vrf_id,
3990                                 rp->addr_only, ~0 /* sw_if_index */ ,
3991                                 rp->proto, !is_delete, rp->twice_nat,
3992                                 rp->out2in_only, rp->tag, rp->identity_nat);
3993   if (rv)
3994     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
3995 }
3996
3997 static void
3998 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3999                                        uword opaque,
4000                                        u32 sw_if_index,
4001                                        ip4_address_t * address,
4002                                        u32 address_length,
4003                                        u32 if_address_index, u32 is_delete)
4004 {
4005   snat_main_t *sm = &snat_main;
4006   snat_static_map_resolve_t *rp;
4007   ip4_address_t l_addr;
4008   int i, j;
4009   int rv;
4010   u8 twice_nat = 0;
4011   snat_address_t *addresses = sm->addresses;
4012
4013   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4014     {
4015       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4016         goto match;
4017     }
4018
4019   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4020     {
4021       twice_nat = 1;
4022       addresses = sm->twice_nat_addresses;
4023       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4024         goto match;
4025     }
4026
4027   return;
4028
4029 match:
4030   if (!is_delete)
4031     {
4032       /* Don't trip over lease renewal, static config */
4033       for (j = 0; j < vec_len (addresses); j++)
4034         if (addresses[j].addr.as_u32 == address->as_u32)
4035           return;
4036
4037       (void) snat_add_address (sm, address, ~0, twice_nat);
4038       /* Scan static map resolution vector */
4039       for (j = 0; j < vec_len (sm->to_resolve); j++)
4040         {
4041           rp = sm->to_resolve + j;
4042           if (rp->addr_only)
4043             continue;
4044           /* On this interface? */
4045           if (rp->sw_if_index == sw_if_index)
4046             {
4047               /* Indetity mapping? */
4048               if (rp->l_addr.as_u32 == 0)
4049                 l_addr.as_u32 = address[0].as_u32;
4050               else
4051                 l_addr.as_u32 = rp->l_addr.as_u32;
4052               /* Add the static mapping */
4053               rv = snat_add_static_mapping (l_addr,
4054                                             address[0],
4055                                             rp->l_port,
4056                                             rp->e_port,
4057                                             rp->vrf_id,
4058                                             rp->addr_only,
4059                                             ~0 /* sw_if_index */ ,
4060                                             rp->proto,
4061                                             rp->is_add, rp->twice_nat,
4062                                             rp->out2in_only, rp->tag,
4063                                             rp->identity_nat);
4064               if (rv)
4065                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4066                                     "i4", rv);
4067             }
4068         }
4069       return;
4070     }
4071   else
4072     {
4073       (void) snat_del_address (sm, address[0], 1, twice_nat);
4074       return;
4075     }
4076 }
4077
4078
4079 int
4080 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4081                             u8 twice_nat)
4082 {
4083   ip4_main_t *ip4_main = sm->ip4_main;
4084   ip4_address_t *first_int_addr;
4085   snat_static_map_resolve_t *rp;
4086   u32 *indices_to_delete = 0;
4087   int i, j;
4088   u32 *auto_add_sw_if_indices =
4089     twice_nat ? sm->
4090     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4091
4092   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4093     );
4094
4095   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4096     {
4097       if (auto_add_sw_if_indices[i] == sw_if_index)
4098         {
4099           if (is_del)
4100             {
4101               /* if have address remove it */
4102               if (first_int_addr)
4103                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4104               else
4105                 {
4106                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4107                     {
4108                       rp = sm->to_resolve + j;
4109                       if (rp->sw_if_index == sw_if_index)
4110                         vec_add1 (indices_to_delete, j);
4111                     }
4112                   if (vec_len (indices_to_delete))
4113                     {
4114                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4115                         vec_del1 (sm->to_resolve, j);
4116                       vec_free (indices_to_delete);
4117                     }
4118                 }
4119               if (twice_nat)
4120                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4121               else
4122                 vec_del1 (sm->auto_add_sw_if_indices, i);
4123             }
4124           else
4125             return VNET_API_ERROR_VALUE_EXIST;
4126
4127           return 0;
4128         }
4129     }
4130
4131   if (is_del)
4132     return VNET_API_ERROR_NO_SUCH_ENTRY;
4133
4134   /* add to the auto-address list */
4135   if (twice_nat)
4136     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4137   else
4138     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4139
4140   /* If the address is already bound - or static - add it now */
4141   if (first_int_addr)
4142     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4143
4144   return 0;
4145 }
4146
4147 int
4148 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4149                    snat_protocol_t proto, u32 vrf_id, int is_in)
4150 {
4151   snat_main_per_thread_data_t *tsm;
4152   clib_bihash_kv_8_8_t kv, value;
4153   ip4_header_t ip;
4154   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4155   snat_session_key_t key;
4156   snat_session_t *s;
4157   clib_bihash_8_8_t *t;
4158
4159   if (sm->endpoint_dependent)
4160     return VNET_API_ERROR_UNSUPPORTED;
4161
4162   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4163   if (sm->num_workers > 1)
4164     tsm =
4165       vec_elt_at_index (sm->per_thread_data,
4166                         sm->worker_in2out_cb (&ip, fib_index, 0));
4167   else
4168     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4169
4170   key.addr.as_u32 = addr->as_u32;
4171   key.port = clib_host_to_net_u16 (port);
4172   key.protocol = proto;
4173   key.fib_index = fib_index;
4174   kv.key = key.as_u64;
4175   t = is_in ? &tsm->in2out : &tsm->out2in;
4176   if (!clib_bihash_search_8_8 (t, &kv, &value))
4177     {
4178       if (pool_is_free_index (tsm->sessions, value.value))
4179         return VNET_API_ERROR_UNSPECIFIED;
4180
4181       s = pool_elt_at_index (tsm->sessions, value.value);
4182       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4183       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4184       return 0;
4185     }
4186
4187   return VNET_API_ERROR_NO_SUCH_ENTRY;
4188 }
4189
4190 int
4191 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4192                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4193                       u32 vrf_id, int is_in)
4194 {
4195   ip4_header_t ip;
4196   clib_bihash_16_8_t *t;
4197   nat_ed_ses_key_t key;
4198   clib_bihash_kv_16_8_t kv, value;
4199   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4200   snat_session_t *s;
4201   snat_main_per_thread_data_t *tsm;
4202
4203   if (!sm->endpoint_dependent)
4204     return VNET_API_ERROR_FEATURE_DISABLED;
4205
4206   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4207   if (sm->num_workers > 1)
4208     tsm =
4209       vec_elt_at_index (sm->per_thread_data,
4210                         sm->worker_in2out_cb (&ip, fib_index, 0));
4211   else
4212     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4213
4214   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4215   key.l_addr.as_u32 = addr->as_u32;
4216   key.r_addr.as_u32 = eh_addr->as_u32;
4217   key.l_port = clib_host_to_net_u16 (port);
4218   key.r_port = clib_host_to_net_u16 (eh_port);
4219   key.proto = proto;
4220   key.fib_index = fib_index;
4221   kv.key[0] = key.as_u64[0];
4222   kv.key[1] = key.as_u64[1];
4223   if (clib_bihash_search_16_8 (t, &kv, &value))
4224     return VNET_API_ERROR_NO_SUCH_ENTRY;
4225
4226   if (pool_is_free_index (tsm->sessions, value.value))
4227     return VNET_API_ERROR_UNSPECIFIED;
4228   s = pool_elt_at_index (tsm->sessions, value.value);
4229   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4230   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4231   return 0;
4232 }
4233
4234 void
4235 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4236 {
4237   snat_main_t *sm = &snat_main;
4238
4239   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4240   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4241   sm->psid = psid;
4242   sm->psid_offset = psid_offset;
4243   sm->psid_length = psid_length;
4244 }
4245
4246 void
4247 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4248 {
4249   snat_main_t *sm = &snat_main;
4250
4251   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4252   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4253   sm->start_port = start_port;
4254   sm->end_port = end_port;
4255 }
4256
4257 void
4258 nat_set_alloc_addr_and_port_default (void)
4259 {
4260   snat_main_t *sm = &snat_main;
4261
4262   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4263   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4264 }
4265
4266 /*
4267  * fd.io coding-style-patch-verification: ON
4268  *
4269  * Local Variables:
4270  * eval: (c-set-style "gnu")
4271  * End:
4272  */