src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36 #include <vppinfra/bihash_16_8.h>
  37
  38 #include <vpp/app/version.h>
  39
  40 snat_main_t snat_main;
  41
  42 fib_source_t nat_fib_src_hi;
  43 fib_source_t nat_fib_src_low;
  44
  45 /* *INDENT-OFF* */
  46 /* Hook up input features */
  47 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  48   .arc_name = "ip4-unicast",
  49   .node_name = "nat-pre-in2out",
  50   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  51                                "ip4-sv-reassembly-feature"),
  52 };
  53 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  54   .arc_name = "ip4-unicast",
  55   .node_name = "nat-pre-out2in",
  56   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  57                                "ip4-dhcp-client-detect",
  58                                "ip4-sv-reassembly-feature"),
  59 };
  60 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  61   .arc_name = "ip4-unicast",
  62   .node_name = "nat44-in2out-worker-handoff",
  63   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  64 };
  65 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  66   .arc_name = "ip4-unicast",
  67   .node_name = "nat44-out2in-worker-handoff",
  68   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  69                                "ip4-dhcp-client-detect"),
  70 };
  71 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  72   .arc_name = "ip4-unicast",
  73   .node_name = "nat44-in2out",
  74   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  75 };
  76 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  77   .arc_name = "ip4-unicast",
  78   .node_name = "nat44-out2in",
  79   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  80                                "ip4-dhcp-client-detect"),
  81 };
  82 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  83   .arc_name = "ip4-unicast",
  84   .node_name = "nat44-classify",
  85   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  86 };
  87 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  88   .arc_name = "ip4-unicast",
  89   .node_name = "nat44-det-in2out",
  90   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-det-out2in",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  96                                "ip4-dhcp-client-detect"),
  97 };
  98 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-det-classify",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 102 };
 103 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-ed-in2out",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-ed-out2in",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 112                                "ip4-dhcp-client-detect"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-ed-classify",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 118 };
 119 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-handoff-classify",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 123 };
 124 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-in2out-fast",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 128 };
 129 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 130   .arc_name = "ip4-unicast",
 131   .node_name = "nat44-out2in-fast",
 132   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 133                                "ip4-dhcp-client-detect"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 136   .arc_name = "ip4-unicast",
 137   .node_name = "nat44-hairpin-dst",
 138   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 141   .arc_name = "ip4-unicast",
 142   .node_name = "nat44-ed-hairpin-dst",
 143   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 144 };
 145
 146 /* Hook up output features */
 147 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 148   .arc_name = "ip4-output",
 149   .node_name = "nat44-in2out-output",
 150   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 151 };
 152 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 153   .arc_name = "ip4-output",
 154   .node_name = "nat44-in2out-output-worker-handoff",
 155   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 156 };
 157 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 158   .arc_name = "ip4-output",
 159   .node_name = "nat44-hairpin-src",
 160   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 161 };
 162 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 163   .arc_name = "ip4-output",
 164   .node_name = "nat44-ed-in2out-output",
 165   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 166   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 167 };
 168 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 169   .arc_name = "ip4-output",
 170   .node_name = "nat44-ed-hairpin-src",
 171   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 172   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 173 };
 174
 175 /* Hook up ip4-local features */
 176 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 177 {
 178   .arc_name = "ip4-local",
 179   .node_name = "nat44-hairpinning",
 180   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 181 };
 182 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 183 {
 184   .arc_name = "ip4-local",
 185   .node_name = "nat44-ed-hairpinning",
 186   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 187 };
 188
 189
 190 VLIB_PLUGIN_REGISTER () = {
 191     .version = VPP_BUILD_VER,
 192     .description = "Network Address Translation (NAT)",
 193 };
 194 /* *INDENT-ON* */
 195
 196 void
 197 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 198                        u8 is_ha)
 199 {
 200   snat_session_key_t key;
 201   clib_bihash_kv_8_8_t kv;
 202   u8 proto;
 203   u16 r_port, l_port;
 204   ip4_address_t *l_addr, *r_addr;
 205   u32 fib_index = 0;
 206   clib_bihash_kv_16_8_t ed_kv;
 207   snat_main_per_thread_data_t *tsm =
 208     vec_elt_at_index (sm->per_thread_data, thread_index);
 209
 210   if (is_fwd_bypass_session (s))
 211     {
 212       if (snat_is_unk_proto_session (s))
 213         {
 214           make_ed_kv (&s->in2out.addr, &s->ext_host_addr, s->in2out.port, 0,
 215                       0, 0, ~0ULL, &ed_kv);
 216         }
 217       else
 218         {
 219           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 220           l_port = s->in2out.port;
 221           r_port = s->ext_host_port;
 222           l_addr = &s->in2out.addr;
 223           r_addr = &s->ext_host_addr;
 224           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 225           make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 226                       &ed_kv);
 227         }
 228       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 229         nat_elog_warn ("in2out_ed key del failed");
 230       return;
 231     }
 232
 233   /* session lookup tables */
 234   if (is_ed_session (s))
 235     {
 236       if (is_affinity_sessions (s))
 237         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 238                              s->in2out.protocol, s->out2in.port);
 239       l_addr = &s->out2in.addr;
 240       r_addr = &s->ext_host_addr;
 241       fib_index = s->out2in.fib_index;
 242       if (snat_is_unk_proto_session (s))
 243         {
 244           proto = s->in2out.port;
 245           r_port = 0;
 246           l_port = 0;
 247         }
 248       else
 249         {
 250           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 251           l_port = s->out2in.port;
 252           r_port = s->ext_host_port;
 253         }
 254       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 255                   &ed_kv);
 256       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 257         nat_elog_warn ("out2in_ed key del failed");
 258       l_addr = &s->in2out.addr;
 259       fib_index = s->in2out.fib_index;
 260       if (!snat_is_unk_proto_session (s))
 261         l_port = s->in2out.port;
 262       if (is_twice_nat_session (s))
 263         {
 264           r_addr = &s->ext_host_nat_addr;
 265           r_port = s->ext_host_nat_port;
 266         }
 267       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 268                   &ed_kv);
 269       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 270         nat_elog_warn ("in2out_ed key del failed");
 271
 272       if (!is_ha)
 273         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 274                                &s->in2out.addr, s->in2out.port,
 275                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 276                                &s->out2in.addr, s->out2in.port,
 277                                &s->ext_host_addr, s->ext_host_port,
 278                                s->in2out.protocol, is_twice_nat_session (s));
 279     }
 280   else
 281     {
 282       kv.key = s->in2out.as_u64;
 283       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 284         nat_elog_warn ("in2out key del failed");
 285       kv.key = s->out2in.as_u64;
 286       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 287         nat_elog_warn ("out2in key del failed");
 288
 289       if (!is_ha)
 290         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 291                                  &s->in2out.addr, s->in2out.port,
 292                                  &s->out2in.addr, s->out2in.port,
 293                                  s->in2out.protocol);
 294     }
 295
 296   if (snat_is_unk_proto_session (s))
 297     return;
 298
 299   if (!is_ha)
 300     {
 301       /* log NAT event */
 302       snat_ipfix_logging_nat44_ses_delete (thread_index,
 303                                            s->in2out.addr.as_u32,
 304                                            s->out2in.addr.as_u32,
 305                                            s->in2out.protocol,
 306                                            s->in2out.port,
 307                                            s->out2in.port,
 308                                            s->in2out.fib_index);
 309
 310       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 311                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 312                    thread_index);
 313     }
 314
 315   /* Twice NAT address and port for external host */
 316   if (is_twice_nat_session (s))
 317     {
 318       key.protocol = s->in2out.protocol;
 319       key.port = s->ext_host_nat_port;
 320       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 321       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 322                                           thread_index, &key);
 323     }
 324
 325   if (snat_is_session_static (s))
 326     return;
 327
 328   snat_free_outside_address_and_port (sm->addresses, thread_index,
 329                                       &s->out2in);
 330 }
 331
 332 int
 333 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
 334 {
 335   snat_main_t *sm = &snat_main;
 336   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
 337   u32 len = vec_len (sm->max_translations_per_fib);
 338
 339   if (len <= fib_index)
 340     {
 341       vec_validate (sm->max_translations_per_fib, fib_index + 1);
 342
 343       for (; len < vec_len (sm->max_translations_per_fib); len++)
 344         sm->max_translations_per_fib[len] = sm->max_translations;
 345     }
 346
 347   sm->max_translations_per_fib[fib_index] = session_limit;
 348   return 0;
 349 }
 350
 351 void
 352 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 353                          u32 thread_index, u8 is_ha)
 354 {
 355   snat_session_key_t key;
 356   u8 proto;
 357   u16 r_port, l_port;
 358   ip4_address_t *l_addr, *r_addr;
 359   u32 fib_index;
 360   clib_bihash_kv_16_8_t ed_kv;
 361   snat_main_per_thread_data_t *tsm =
 362     vec_elt_at_index (sm->per_thread_data, thread_index);
 363
 364   if (is_fwd_bypass_session (s))
 365     {
 366       if (snat_is_unk_proto_session (s))
 367         {
 368           proto = s->in2out.port;
 369           r_port = 0;
 370           l_port = 0;
 371         }
 372       else
 373         {
 374           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 375           l_port = s->in2out.port;
 376           r_port = s->ext_host_port;
 377         }
 378
 379       l_addr = &s->in2out.addr;
 380       r_addr = &s->ext_host_addr;
 381       fib_index = 0;
 382       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 383                   &ed_kv);
 384
 385       if (PREDICT_FALSE
 386           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 387         nat_elog_warn ("in2out_ed key del failed");
 388       return;
 389     }
 390
 391   /* session lookup tables */
 392   if (is_affinity_sessions (s))
 393     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 394                          s->in2out.protocol, s->out2in.port);
 395   l_addr = &s->out2in.addr;
 396   r_addr = &s->ext_host_addr;
 397   fib_index = s->out2in.fib_index;
 398   if (snat_is_unk_proto_session (s))
 399     {
 400       proto = s->in2out.port;
 401       r_port = 0;
 402       l_port = 0;
 403     }
 404   else
 405     {
 406       proto = snat_proto_to_ip_proto (s->in2out.protocol);
 407       l_port = s->out2in.port;
 408       r_port = s->ext_host_port;
 409     }
 410   make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 411               &ed_kv);
 412
 413   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0)))
 414     nat_elog_warn ("out2in_ed key del failed");
 415
 416   l_addr = &s->in2out.addr;
 417   fib_index = s->in2out.fib_index;
 418
 419   if (!snat_is_unk_proto_session (s))
 420     l_port = s->in2out.port;
 421
 422   if (is_twice_nat_session (s))
 423     {
 424       r_addr = &s->ext_host_nat_addr;
 425       r_port = s->ext_host_nat_port;
 426     }
 427   make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 428               &ed_kv);
 429
 430   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 431     nat_elog_warn ("in2out_ed key del failed");
 432
 433   if (!is_ha)
 434     {
 435       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 436                              &s->in2out.addr, s->in2out.port,
 437                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 438                              &s->out2in.addr, s->out2in.port,
 439                              &s->ext_host_addr, s->ext_host_port,
 440                              s->in2out.protocol, is_twice_nat_session (s));
 441     }
 442
 443   if (snat_is_unk_proto_session (s))
 444     return;
 445
 446   // is this correct ?
 447   if (!is_ha)
 448     {
 449       snat_ipfix_logging_nat44_ses_delete (thread_index,
 450                                            s->in2out.addr.as_u32,
 451                                            s->out2in.addr.as_u32,
 452                                            s->in2out.protocol,
 453                                            s->in2out.port,
 454                                            s->out2in.port,
 455                                            s->in2out.fib_index);
 456       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 457                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 458                    thread_index);
 459     }
 460
 461   /* Twice NAT address and port for external host */
 462   if (is_twice_nat_session (s))
 463     {
 464       key.protocol = s->in2out.protocol;
 465       key.port = s->ext_host_nat_port;
 466       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 467       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 468                                           thread_index, &key);
 469     }
 470
 471   if (snat_is_session_static (s))
 472     return;
 473
 474   // should be called for every dynamic session
 475   snat_free_outside_address_and_port (sm->addresses, thread_index,
 476                                       &s->out2in);
 477 }
 478
 479
 480 snat_user_t *
 481 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 482                         u32 thread_index)
 483 {
 484   snat_user_t *u = 0;
 485   snat_user_key_t user_key;
 486   clib_bihash_kv_8_8_t kv, value;
 487   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 488   dlist_elt_t *per_user_list_head_elt;
 489
 490   user_key.addr.as_u32 = addr->as_u32;
 491   user_key.fib_index = fib_index;
 492   kv.key = user_key.as_u64;
 493
 494   /* Ever heard of the "user" = src ip4 address before? */
 495   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 496     {
 497       /* no, make a new one */
 498       pool_get (tsm->users, u);
 499       clib_memset (u, 0, sizeof (*u));
 500
 501       u->addr.as_u32 = addr->as_u32;
 502       u->fib_index = fib_index;
 503
 504       pool_get (tsm->list_pool, per_user_list_head_elt);
 505
 506       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 507         tsm->list_pool;
 508
 509       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 510
 511       kv.value = u - tsm->users;
 512
 513       /* add user */
 514       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 515         {
 516           nat_elog_warn ("user_hash key add failed");
 517           nat44_delete_user_with_no_session (sm, u, thread_index);
 518           return NULL;
 519         }
 520
 521       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 522                                pool_elts (tsm->users));
 523     }
 524   else
 525     {
 526       u = pool_elt_at_index (tsm->users, value.value);
 527     }
 528
 529   return u;
 530 }
 531
 532 snat_session_t *
 533 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 534                               u32 thread_index, f64 now)
 535 {
 536   snat_session_t *s;
 537   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 538   u32 oldest_per_user_translation_list_index, session_index;
 539   dlist_elt_t *oldest_per_user_translation_list_elt;
 540   dlist_elt_t *per_user_translation_list_elt;
 541
 542   /* Over quota? Recycle the least recently used translation */
 543   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 544     {
 545       oldest_per_user_translation_list_index =
 546         clib_dlist_remove_head (tsm->list_pool,
 547                                 u->sessions_per_user_list_head_index);
 548
 549       ASSERT (oldest_per_user_translation_list_index != ~0);
 550
 551       /* Add it back to the end of the LRU list */
 552       clib_dlist_addtail (tsm->list_pool,
 553                           u->sessions_per_user_list_head_index,
 554                           oldest_per_user_translation_list_index);
 555       /* Get the list element */
 556       oldest_per_user_translation_list_elt =
 557         pool_elt_at_index (tsm->list_pool,
 558                            oldest_per_user_translation_list_index);
 559
 560       /* Get the session index from the list element */
 561       session_index = oldest_per_user_translation_list_elt->value;
 562
 563       /* Get the session */
 564       s = pool_elt_at_index (tsm->sessions, session_index);
 565       nat_free_session_data (sm, s, thread_index, 0);
 566       if (snat_is_session_static (s))
 567         u->nstaticsessions--;
 568       else
 569         u->nsessions--;
 570       s->flags = 0;
 571       s->total_bytes = 0;
 572       s->total_pkts = 0;
 573       s->state = 0;
 574       s->ext_host_addr.as_u32 = 0;
 575       s->ext_host_port = 0;
 576       s->ext_host_nat_addr.as_u32 = 0;
 577       s->ext_host_nat_port = 0;
 578     }
 579   else
 580     {
 581       pool_get (tsm->sessions, s);
 582       clib_memset (s, 0, sizeof (*s));
 583
 584       /* Create list elts */
 585       pool_get (tsm->list_pool, per_user_translation_list_elt);
 586       clib_dlist_init (tsm->list_pool,
 587                        per_user_translation_list_elt - tsm->list_pool);
 588
 589       per_user_translation_list_elt->value = s - tsm->sessions;
 590       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 591       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 592
 593       clib_dlist_addtail (tsm->list_pool,
 594                           s->per_user_list_head_index,
 595                           per_user_translation_list_elt - tsm->list_pool);
 596
 597       dlist_elt_t *global_lru_list_elt;
 598       pool_get (tsm->global_lru_pool, global_lru_list_elt);
 599       global_lru_list_elt->value = s - tsm->sessions;
 600       s->global_lru_index = global_lru_list_elt - tsm->global_lru_pool;
 601       clib_dlist_addtail (tsm->global_lru_pool, tsm->global_lru_head_index,
 602                           s->global_lru_index);
 603       s->last_lru_update = now;
 604
 605       s->user_index = u - tsm->users;
 606       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 607                                pool_elts (tsm->sessions));
 608     }
 609
 610   s->ha_last_refreshed = now;
 611
 612   return s;
 613 }
 614
 615 int
 616 nat_global_lru_free_one (snat_main_t * sm, int thread_index, f64 now)
 617 {
 618   snat_session_t *s = NULL;
 619   dlist_elt_t *oldest_elt;
 620   u64 sess_timeout_time;
 621   u32 oldest_index;
 622   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 623   oldest_index = clib_dlist_remove_head (tsm->global_lru_pool,
 624                                          tsm->global_lru_head_index);
 625   if (~0 != oldest_index)
 626     {
 627       oldest_elt = pool_elt_at_index (tsm->global_lru_pool, oldest_index);
 628       s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 629
 630       sess_timeout_time =
 631         s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 632       if (now >= sess_timeout_time
 633           || (s->tcp_close_timestamp && now >= s->tcp_close_timestamp))
 634         {
 635           nat_free_session_data (sm, s, thread_index, 0);
 636           nat44_ed_delete_session (sm, s, thread_index, 0);
 637           return 1;
 638         }
 639       else
 640         {
 641           clib_dlist_addhead (tsm->global_lru_pool,
 642                               tsm->global_lru_head_index, oldest_index);
 643         }
 644     }
 645   return 0;
 646 }
 647
 648 snat_session_t *
 649 nat_ed_session_alloc (snat_main_t * sm, u32 thread_index, f64 now)
 650 {
 651   snat_session_t *s;
 652   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 653
 654   nat_global_lru_free_one (sm, thread_index, now);
 655
 656   pool_get (tsm->sessions, s);
 657   clib_memset (s, 0, sizeof (*s));
 658
 659   nat44_global_lru_insert (tsm, s, now);
 660
 661   s->ha_last_refreshed = now;
 662   vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 663                            pool_elts (tsm->sessions));
 664   return s;
 665 }
 666
 667 void
 668 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 669                           int is_add)
 670 {
 671   fib_prefix_t prefix = {
 672     .fp_len = p_len,
 673     .fp_proto = FIB_PROTOCOL_IP4,
 674     .fp_addr = {
 675                 .ip4.as_u32 = addr->as_u32,
 676                 },
 677   };
 678   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 679
 680   if (is_add)
 681     fib_table_entry_update_one_path (fib_index,
 682                                      &prefix,
 683                                      nat_fib_src_low,
 684                                      (FIB_ENTRY_FLAG_CONNECTED |
 685                                       FIB_ENTRY_FLAG_LOCAL |
 686                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 687                                      DPO_PROTO_IP4,
 688                                      NULL,
 689                                      sw_if_index,
 690                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 691   else
 692     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 693 }
 694
 695 int
 696 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 697                   u8 twice_nat)
 698 {
 699   snat_address_t *ap;
 700   snat_interface_t *i;
 701   vlib_thread_main_t *tm = vlib_get_thread_main ();
 702
 703   if (twice_nat && !sm->endpoint_dependent)
 704     return VNET_API_ERROR_FEATURE_DISABLED;
 705
 706   /* Check if address already exists */
 707   /* *INDENT-OFF* */
 708   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 709     {
 710       if (ap->addr.as_u32 == addr->as_u32)
 711         return VNET_API_ERROR_VALUE_EXIST;
 712     }
 713   /* *INDENT-ON* */
 714
 715   if (twice_nat)
 716     vec_add2 (sm->twice_nat_addresses, ap, 1);
 717   else
 718     vec_add2 (sm->addresses, ap, 1);
 719
 720   ap->addr = *addr;
 721   if (vrf_id != ~0)
 722     ap->fib_index =
 723       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 724                                          nat_fib_src_low);
 725   else
 726     ap->fib_index = ~0;
 727 #define _(N, i, n, s) \
 728   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 729   ap->busy_##n##_ports = 0; \
 730   ap->busy_##n##_ports_per_thread = 0;\
 731   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 732   foreach_snat_protocol
 733 #undef _
 734     if (twice_nat)
 735     return 0;
 736
 737   /* Add external address to FIB */
 738   /* *INDENT-OFF* */
 739   pool_foreach (i, sm->interfaces,
 740   ({
 741     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 742       continue;
 743
 744     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 745     break;
 746   }));
 747   pool_foreach (i, sm->output_feature_interfaces,
 748   ({
 749     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 750       continue;
 751
 752     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 753     break;
 754   }));
 755   /* *INDENT-ON* */
 756
 757   return 0;
 758 }
 759
 760 static int
 761 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 762 {
 763   snat_static_mapping_t *m;
 764   /* *INDENT-OFF* */
 765   pool_foreach (m, sm->static_mappings,
 766   ({
 767       if (is_addr_only_static_mapping (m) ||
 768           is_out2in_only_static_mapping (m) ||
 769           is_identity_static_mapping (m))
 770         continue;
 771       if (m->external_addr.as_u32 == addr.as_u32)
 772         return 1;
 773   }));
 774   /* *INDENT-ON* */
 775
 776   return 0;
 777 }
 778
 779 static void
 780 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 781                                        ip4_address_t l_addr,
 782                                        u16 l_port,
 783                                        u32 sw_if_index,
 784                                        u16 e_port,
 785                                        u32 vrf_id,
 786                                        snat_protocol_t proto,
 787                                        int addr_only, int is_add, u8 * tag,
 788                                        int twice_nat, int out2in_only,
 789                                        int identity_nat)
 790 {
 791   snat_static_map_resolve_t *rp;
 792
 793   vec_add2 (sm->to_resolve, rp, 1);
 794   rp->l_addr.as_u32 = l_addr.as_u32;
 795   rp->l_port = l_port;
 796   rp->sw_if_index = sw_if_index;
 797   rp->e_port = e_port;
 798   rp->vrf_id = vrf_id;
 799   rp->proto = proto;
 800   rp->addr_only = addr_only;
 801   rp->is_add = is_add;
 802   rp->twice_nat = twice_nat;
 803   rp->out2in_only = out2in_only;
 804   rp->identity_nat = identity_nat;
 805   rp->tag = vec_dup (tag);
 806 }
 807
 808 static u32
 809 get_thread_idx_by_port (u16 e_port)
 810 {
 811   snat_main_t *sm = &snat_main;
 812   u32 thread_idx = sm->num_workers;
 813   if (sm->num_workers > 1)
 814     {
 815       thread_idx =
 816         sm->first_worker_index +
 817         sm->workers[(e_port - 1024) / sm->port_per_thread];
 818     }
 819   return thread_idx;
 820 }
 821
 822 void
 823 snat_static_mapping_del_sessions (snat_main_t * sm,
 824                                   snat_main_per_thread_data_t * tsm,
 825                                   snat_user_key_t u_key, int addr_only,
 826                                   ip4_address_t e_addr, u16 e_port)
 827 {
 828   clib_bihash_kv_8_8_t kv, value;
 829   kv.key = u_key.as_u64;
 830   u64 user_index;
 831   dlist_elt_t *head, *elt;
 832   snat_user_t *u;
 833   snat_session_t *s;
 834   u32 elt_index, head_index, ses_index;
 835   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 836     {
 837       user_index = value.value;
 838       u = pool_elt_at_index (tsm->users, user_index);
 839       if (u->nstaticsessions)
 840         {
 841           head_index = u->sessions_per_user_list_head_index;
 842           head = pool_elt_at_index (tsm->list_pool, head_index);
 843           elt_index = head->next;
 844           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 845           ses_index = elt->value;
 846           while (ses_index != ~0)
 847             {
 848               s = pool_elt_at_index (tsm->sessions, ses_index);
 849               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 850               ses_index = elt->value;
 851
 852               if (!addr_only)
 853                 {
 854                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 855                       (clib_net_to_host_u16 (s->out2in.port) != e_port))
 856                     continue;
 857                 }
 858
 859               if (is_lb_session (s))
 860                 continue;
 861
 862               if (!snat_is_session_static (s))
 863                 continue;
 864
 865               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 866               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 867
 868               if (!addr_only)
 869                 break;
 870             }
 871         }
 872     }
 873 }
 874
 875 void
 876 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 877                                      snat_main_per_thread_data_t * tsm,
 878                                      ip4_address_t l_addr,
 879                                      u16 l_port,
 880                                      u8 protocol,
 881                                      u32 fib_index, int addr_only,
 882                                      ip4_address_t e_addr, u16 e_port)
 883 {
 884   snat_session_t *s;
 885   u32 *indexes_to_free = NULL;
 886   /* *INDENT-OFF* */
 887   pool_foreach (s, tsm->sessions, {
 888     if (s->in2out.fib_index != fib_index ||
 889         s->in2out.addr.as_u32 != l_addr.as_u32)
 890       {
 891         continue;
 892       }
 893     if (!addr_only)
 894       {
 895         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 896             (clib_net_to_host_u16 (s->out2in.port) != e_port) ||
 897             clib_net_to_host_u16 (s->in2out.port) != l_port ||
 898             s->in2out.protocol != protocol)
 899           continue;
 900       }
 901
 902     if (is_lb_session (s))
 903       continue;
 904     if (!snat_is_session_static (s))
 905       continue;
 906     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 907     vec_add1 (indexes_to_free, s - tsm->sessions);
 908     if (!addr_only)
 909       break;
 910   });
 911   /* *INDENT-ON* */
 912   u32 *ses_index;
 913   vec_foreach (ses_index, indexes_to_free)
 914   {
 915     s = pool_elt_at_index (tsm->sessions, *ses_index);
 916     nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
 917   }
 918   vec_free (indexes_to_free);
 919 }
 920
 921 int
 922 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 923                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 924                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 925                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 926                          u8 identity_nat)
 927 {
 928   snat_main_t *sm = &snat_main;
 929   snat_static_mapping_t *m;
 930   snat_session_key_t m_key;
 931   clib_bihash_kv_8_8_t kv, value;
 932   snat_address_t *a = 0;
 933   u32 fib_index = ~0;
 934   snat_interface_t *interface;
 935   int i;
 936   snat_main_per_thread_data_t *tsm;
 937   snat_user_key_t u_key;
 938   snat_user_t *u;
 939   dlist_elt_t *head, *elt;
 940   u32 elt_index, head_index;
 941   u32 ses_index;
 942   u64 user_index;
 943   snat_session_t *s;
 944   snat_static_map_resolve_t *rp, *rp_match = 0;
 945   nat44_lb_addr_port_t *local;
 946   u32 find = ~0;
 947
 948   if (!sm->endpoint_dependent)
 949     {
 950       if (twice_nat || out2in_only)
 951         return VNET_API_ERROR_FEATURE_DISABLED;
 952     }
 953
 954   /* If the external address is a specific interface address */
 955   if (sw_if_index != ~0)
 956     {
 957       ip4_address_t *first_int_addr;
 958
 959       for (i = 0; i < vec_len (sm->to_resolve); i++)
 960         {
 961           rp = sm->to_resolve + i;
 962           if (rp->sw_if_index != sw_if_index ||
 963               rp->l_addr.as_u32 != l_addr.as_u32 ||
 964               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 965             continue;
 966
 967           if (!addr_only)
 968             {
 969               if ((rp->l_port != l_port && rp->e_port != e_port)
 970                   || rp->proto != proto)
 971                 continue;
 972             }
 973
 974           rp_match = rp;
 975           break;
 976         }
 977
 978       /* Might be already set... */
 979       first_int_addr = ip4_interface_first_address
 980         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 981
 982       if (is_add)
 983         {
 984           if (rp_match)
 985             return VNET_API_ERROR_VALUE_EXIST;
 986
 987           snat_add_static_mapping_when_resolved
 988             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 989              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 990
 991           /* DHCP resolution required? */
 992           if (first_int_addr == 0)
 993             {
 994               return 0;
 995             }
 996           else
 997             {
 998               e_addr.as_u32 = first_int_addr->as_u32;
 999               /* Identity mapping? */
1000               if (l_addr.as_u32 == 0)
1001                 l_addr.as_u32 = e_addr.as_u32;
1002             }
1003         }
1004       else
1005         {
1006           if (!rp_match)
1007             return VNET_API_ERROR_NO_SUCH_ENTRY;
1008
1009           vec_del1 (sm->to_resolve, i);
1010
1011           if (first_int_addr)
1012             {
1013               e_addr.as_u32 = first_int_addr->as_u32;
1014               /* Identity mapping? */
1015               if (l_addr.as_u32 == 0)
1016                 l_addr.as_u32 = e_addr.as_u32;
1017             }
1018           else
1019             return 0;
1020         }
1021     }
1022
1023   m_key.addr = e_addr;
1024   m_key.port = addr_only ? 0 : e_port;
1025   m_key.protocol = addr_only ? 0 : proto;
1026   m_key.fib_index = 0;
1027   kv.key = m_key.as_u64;
1028   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1029     m = 0;
1030   else
1031     m = pool_elt_at_index (sm->static_mappings, value.value);
1032
1033   if (is_add)
1034     {
1035       if (m)
1036         {
1037           if (is_identity_static_mapping (m))
1038             {
1039               /* *INDENT-OFF* */
1040               pool_foreach (local, m->locals,
1041               ({
1042                 if (local->vrf_id == vrf_id)
1043                   return VNET_API_ERROR_VALUE_EXIST;
1044               }));
1045               /* *INDENT-ON* */
1046               pool_get (m->locals, local);
1047               local->vrf_id = vrf_id;
1048               local->fib_index =
1049                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1050                                                    nat_fib_src_low);
1051               m_key.addr = m->local_addr;
1052               m_key.port = m->local_port;
1053               m_key.protocol = m->proto;
1054               m_key.fib_index = local->fib_index;
1055               kv.key = m_key.as_u64;
1056               kv.value = m - sm->static_mappings;
1057               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1058               return 0;
1059             }
1060           else
1061             return VNET_API_ERROR_VALUE_EXIST;
1062         }
1063
1064       if (twice_nat && addr_only)
1065         return VNET_API_ERROR_UNSUPPORTED;
1066
1067       /* Convert VRF id to FIB index */
1068       if (vrf_id != ~0)
1069         fib_index =
1070           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1071                                              nat_fib_src_low);
1072       /* If not specified use inside VRF id from SNAT plugin startup config */
1073       else
1074         {
1075           fib_index = sm->inside_fib_index;
1076           vrf_id = sm->inside_vrf_id;
1077           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1078         }
1079
1080       if (!(out2in_only || identity_nat))
1081         {
1082           m_key.addr = l_addr;
1083           m_key.port = addr_only ? 0 : l_port;
1084           m_key.protocol = addr_only ? 0 : proto;
1085           m_key.fib_index = fib_index;
1086           kv.key = m_key.as_u64;
1087           if (!clib_bihash_search_8_8
1088               (&sm->static_mapping_by_local, &kv, &value))
1089             return VNET_API_ERROR_VALUE_EXIST;
1090         }
1091
1092       /* Find external address in allocated addresses and reserve port for
1093          address and port pair mapping when dynamic translations enabled */
1094       if (!(addr_only || sm->static_mapping_only || out2in_only))
1095         {
1096           for (i = 0; i < vec_len (sm->addresses); i++)
1097             {
1098               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1099                 {
1100                   a = sm->addresses + i;
1101                   /* External port must be unused */
1102                   switch (proto)
1103                     {
1104 #define _(N, j, n, s) \
1105                     case SNAT_PROTOCOL_##N: \
1106                       if (a->busy_##n##_port_refcounts[e_port]) \
1107                         return VNET_API_ERROR_INVALID_VALUE; \
1108                       ++a->busy_##n##_port_refcounts[e_port]; \
1109                       if (e_port > 1024) \
1110                         { \
1111                           a->busy_##n##_ports++; \
1112                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1113                         } \
1114                       break;
1115                       foreach_snat_protocol
1116 #undef _
1117                     default:
1118                       nat_elog_info ("unknown protocol");
1119                       return VNET_API_ERROR_INVALID_VALUE_2;
1120                     }
1121                   break;
1122                 }
1123             }
1124           /* External address must be allocated */
1125           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1126             {
1127               if (sw_if_index != ~0)
1128                 {
1129                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1130                     {
1131                       rp = sm->to_resolve + i;
1132                       if (rp->addr_only)
1133                         continue;
1134                       if (rp->sw_if_index != sw_if_index &&
1135                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1136                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1137                           rp->e_port != e_port && rp->proto != proto)
1138                         continue;
1139
1140                       vec_del1 (sm->to_resolve, i);
1141                       break;
1142                     }
1143                 }
1144               return VNET_API_ERROR_NO_SUCH_ENTRY;
1145             }
1146         }
1147
1148       pool_get (sm->static_mappings, m);
1149       clib_memset (m, 0, sizeof (*m));
1150       m->tag = vec_dup (tag);
1151       m->local_addr = l_addr;
1152       m->external_addr = e_addr;
1153       m->twice_nat = twice_nat;
1154       if (out2in_only)
1155         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1156       if (addr_only)
1157         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1158       if (identity_nat)
1159         {
1160           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1161           pool_get (m->locals, local);
1162           local->vrf_id = vrf_id;
1163           local->fib_index = fib_index;
1164         }
1165       else
1166         {
1167           m->vrf_id = vrf_id;
1168           m->fib_index = fib_index;
1169         }
1170       if (!addr_only)
1171         {
1172           m->local_port = l_port;
1173           m->external_port = e_port;
1174           m->proto = proto;
1175         }
1176
1177       if (sm->num_workers > 1)
1178         {
1179           ip4_header_t ip = {
1180             .src_address = m->local_addr,
1181           };
1182           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1183           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1184         }
1185       else
1186         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1187
1188       m_key.addr = m->local_addr;
1189       m_key.port = m->local_port;
1190       m_key.protocol = m->proto;
1191       m_key.fib_index = fib_index;
1192       kv.key = m_key.as_u64;
1193       kv.value = m - sm->static_mappings;
1194       if (!out2in_only)
1195         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1196
1197       m_key.addr = m->external_addr;
1198       m_key.port = m->external_port;
1199       m_key.fib_index = 0;
1200       kv.key = m_key.as_u64;
1201       kv.value = m - sm->static_mappings;
1202       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1203
1204       /* Delete dynamic sessions matching local address (+ local port) */
1205       if (!(sm->static_mapping_only))
1206         {
1207           u_key.addr = m->local_addr;
1208           u_key.fib_index = m->fib_index;
1209           kv.key = u_key.as_u64;
1210           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1211             {
1212               user_index = value.value;
1213               u = pool_elt_at_index (tsm->users, user_index);
1214               if (u->nsessions)
1215                 {
1216                   head_index = u->sessions_per_user_list_head_index;
1217                   head = pool_elt_at_index (tsm->list_pool, head_index);
1218                   elt_index = head->next;
1219                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1220                   ses_index = elt->value;
1221                   while (ses_index != ~0)
1222                     {
1223                       s = pool_elt_at_index (tsm->sessions, ses_index);
1224                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1225                       ses_index = elt->value;
1226
1227                       if (snat_is_session_static (s))
1228                         continue;
1229
1230                       if (!addr_only
1231                           && (clib_net_to_host_u16 (s->in2out.port) !=
1232                               m->local_port))
1233                         continue;
1234
1235                       nat_free_session_data (sm, s,
1236                                              tsm - sm->per_thread_data, 0);
1237                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1238
1239                       if (!addr_only && !sm->endpoint_dependent)
1240                         break;
1241                     }
1242                 }
1243             }
1244         }
1245     }
1246   else
1247     {
1248       if (!m)
1249         {
1250           if (sw_if_index != ~0)
1251             return 0;
1252           else
1253             return VNET_API_ERROR_NO_SUCH_ENTRY;
1254         }
1255
1256       if (identity_nat)
1257         {
1258           if (vrf_id == ~0)
1259             vrf_id = sm->inside_vrf_id;
1260
1261           /* *INDENT-OFF* */
1262           pool_foreach (local, m->locals,
1263           ({
1264             if (local->vrf_id == vrf_id)
1265               find = local - m->locals;
1266           }));
1267           /* *INDENT-ON* */
1268           if (find == ~0)
1269             return VNET_API_ERROR_NO_SUCH_ENTRY;
1270
1271           local = pool_elt_at_index (m->locals, find);
1272           fib_index = local->fib_index;
1273           pool_put (m->locals, local);
1274         }
1275       else
1276         fib_index = m->fib_index;
1277
1278       /* Free external address port */
1279       if (!(addr_only || sm->static_mapping_only || out2in_only))
1280         {
1281           for (i = 0; i < vec_len (sm->addresses); i++)
1282             {
1283               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1284                 {
1285                   a = sm->addresses + i;
1286                   switch (proto)
1287                     {
1288 #define _(N, j, n, s) \
1289                     case SNAT_PROTOCOL_##N: \
1290                       --a->busy_##n##_port_refcounts[e_port]; \
1291                       if (e_port > 1024) \
1292                         { \
1293                           a->busy_##n##_ports--; \
1294                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1295                         } \
1296                       break;
1297                       foreach_snat_protocol
1298 #undef _
1299                     default:
1300                       nat_elog_info ("unknown protocol");
1301                       return VNET_API_ERROR_INVALID_VALUE_2;
1302                     }
1303                   break;
1304                 }
1305             }
1306         }
1307
1308       if (sm->num_workers > 1)
1309         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1310       else
1311         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1312
1313       m_key.addr = m->local_addr;
1314       m_key.port = m->local_port;
1315       m_key.protocol = m->proto;
1316       m_key.fib_index = fib_index;
1317       kv.key = m_key.as_u64;
1318       if (!out2in_only)
1319         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1320
1321       /* Delete session(s) for static mapping if exist */
1322       if (!(sm->static_mapping_only) ||
1323           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1324         {
1325           if (sm->endpoint_dependent)
1326             {
1327               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1328                                                    m->local_port, m->proto,
1329                                                    fib_index, addr_only,
1330                                                    e_addr, e_port);
1331             }
1332           else
1333             {
1334               u_key.addr = m->local_addr;
1335               u_key.fib_index = fib_index;
1336               kv.key = u_key.as_u64;
1337               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1338                                                 e_addr, e_port);
1339             }
1340         }
1341
1342       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1343       if (pool_elts (m->locals))
1344         return 0;
1345
1346       m_key.addr = m->external_addr;
1347       m_key.port = m->external_port;
1348       m_key.fib_index = 0;
1349       kv.key = m_key.as_u64;
1350       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1351
1352       vec_free (m->tag);
1353       vec_free (m->workers);
1354       /* Delete static mapping from pool */
1355       pool_put (sm->static_mappings, m);
1356     }
1357
1358   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1359     return 0;
1360
1361   /* Add/delete external address to FIB */
1362   /* *INDENT-OFF* */
1363   pool_foreach (interface, sm->interfaces,
1364   ({
1365     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1366       continue;
1367
1368     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1369     break;
1370   }));
1371   pool_foreach (interface, sm->output_feature_interfaces,
1372   ({
1373     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1374       continue;
1375
1376     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1377     break;
1378   }));
1379   /* *INDENT-ON* */
1380
1381   return 0;
1382 }
1383
1384 int
1385 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1386                                  snat_protocol_t proto,
1387                                  nat44_lb_addr_port_t * locals, u8 is_add,
1388                                  twice_nat_type_t twice_nat, u8 out2in_only,
1389                                  u8 * tag, u32 affinity)
1390 {
1391   snat_main_t *sm = &snat_main;
1392   snat_static_mapping_t *m;
1393   snat_session_key_t m_key;
1394   clib_bihash_kv_8_8_t kv, value;
1395   snat_address_t *a = 0;
1396   int i;
1397   nat44_lb_addr_port_t *local;
1398   snat_main_per_thread_data_t *tsm;
1399   snat_session_t *s;
1400   uword *bitmap = 0;
1401
1402   if (!sm->endpoint_dependent)
1403     return VNET_API_ERROR_FEATURE_DISABLED;
1404
1405   m_key.addr = e_addr;
1406   m_key.port = e_port;
1407   m_key.protocol = proto;
1408   m_key.fib_index = 0;
1409   kv.key = m_key.as_u64;
1410   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1411     m = 0;
1412   else
1413     m = pool_elt_at_index (sm->static_mappings, value.value);
1414
1415   if (is_add)
1416     {
1417       if (m)
1418         return VNET_API_ERROR_VALUE_EXIST;
1419
1420       if (vec_len (locals) < 2)
1421         return VNET_API_ERROR_INVALID_VALUE;
1422
1423       /* Find external address in allocated addresses and reserve port for
1424          address and port pair mapping when dynamic translations enabled */
1425       if (!(sm->static_mapping_only || out2in_only))
1426         {
1427           for (i = 0; i < vec_len (sm->addresses); i++)
1428             {
1429               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1430                 {
1431                   a = sm->addresses + i;
1432                   /* External port must be unused */
1433                   switch (proto)
1434                     {
1435 #define _(N, j, n, s) \
1436                     case SNAT_PROTOCOL_##N: \
1437                       if (a->busy_##n##_port_refcounts[e_port]) \
1438                         return VNET_API_ERROR_INVALID_VALUE; \
1439                       ++a->busy_##n##_port_refcounts[e_port]; \
1440                       if (e_port > 1024) \
1441                         { \
1442                           a->busy_##n##_ports++; \
1443                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1444                         } \
1445                       break;
1446                       foreach_snat_protocol
1447 #undef _
1448                     default:
1449                       nat_elog_info ("unknown protocol");
1450                       return VNET_API_ERROR_INVALID_VALUE_2;
1451                     }
1452                   break;
1453                 }
1454             }
1455           /* External address must be allocated */
1456           if (!a)
1457             return VNET_API_ERROR_NO_SUCH_ENTRY;
1458         }
1459
1460       pool_get (sm->static_mappings, m);
1461       clib_memset (m, 0, sizeof (*m));
1462       m->tag = vec_dup (tag);
1463       m->external_addr = e_addr;
1464       m->external_port = e_port;
1465       m->proto = proto;
1466       m->twice_nat = twice_nat;
1467       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1468       if (out2in_only)
1469         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1470       m->affinity = affinity;
1471
1472       if (affinity)
1473         m->affinity_per_service_list_head_index =
1474           nat_affinity_get_per_service_list_head_index ();
1475       else
1476         m->affinity_per_service_list_head_index = ~0;
1477
1478       m_key.addr = m->external_addr;
1479       m_key.port = m->external_port;
1480       m_key.protocol = m->proto;
1481       m_key.fib_index = 0;
1482       kv.key = m_key.as_u64;
1483       kv.value = m - sm->static_mappings;
1484       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1485         {
1486           nat_elog_err ("static_mapping_by_external key add failed");
1487           return VNET_API_ERROR_UNSPECIFIED;
1488         }
1489
1490       m_key.fib_index = m->fib_index;
1491       for (i = 0; i < vec_len (locals); i++)
1492         {
1493           locals[i].fib_index =
1494             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1495                                                locals[i].vrf_id,
1496                                                nat_fib_src_low);
1497           m_key.addr = locals[i].addr;
1498           m_key.fib_index = locals[i].fib_index;
1499           if (!out2in_only)
1500             {
1501               m_key.port = locals[i].port;
1502               kv.key = m_key.as_u64;
1503               kv.value = m - sm->static_mappings;
1504               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1505             }
1506           locals[i].prefix = (i == 0) ? locals[i].probability :
1507             (locals[i - 1].prefix + locals[i].probability);
1508           pool_get (m->locals, local);
1509           *local = locals[i];
1510           if (sm->num_workers > 1)
1511             {
1512               ip4_header_t ip = {
1513                 .src_address = locals[i].addr,
1514               };
1515               bitmap =
1516                 clib_bitmap_set (bitmap,
1517                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1518                                  1);
1519             }
1520         }
1521
1522       /* Assign workers */
1523       if (sm->num_workers > 1)
1524         {
1525           /* *INDENT-OFF* */
1526           clib_bitmap_foreach (i, bitmap,
1527             ({
1528                vec_add1(m->workers, i);
1529             }));
1530           /* *INDENT-ON* */
1531         }
1532     }
1533   else
1534     {
1535       if (!m)
1536         return VNET_API_ERROR_NO_SUCH_ENTRY;
1537
1538       if (!is_lb_static_mapping (m))
1539         return VNET_API_ERROR_INVALID_VALUE;
1540
1541       /* Free external address port */
1542       if (!(sm->static_mapping_only || out2in_only))
1543         {
1544           for (i = 0; i < vec_len (sm->addresses); i++)
1545             {
1546               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1547                 {
1548                   a = sm->addresses + i;
1549                   switch (proto)
1550                     {
1551 #define _(N, j, n, s) \
1552                     case SNAT_PROTOCOL_##N: \
1553                       --a->busy_##n##_port_refcounts[e_port]; \
1554                       if (e_port > 1024) \
1555                         { \
1556                           a->busy_##n##_ports--; \
1557                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1558                         } \
1559                       break;
1560                       foreach_snat_protocol
1561 #undef _
1562                     default:
1563                       nat_elog_info ("unknown protocol");
1564                       return VNET_API_ERROR_INVALID_VALUE_2;
1565                     }
1566                   break;
1567                 }
1568             }
1569         }
1570
1571       m_key.addr = m->external_addr;
1572       m_key.port = m->external_port;
1573       m_key.protocol = m->proto;
1574       m_key.fib_index = 0;
1575       kv.key = m_key.as_u64;
1576       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1577         {
1578           nat_elog_err ("static_mapping_by_external key del failed");
1579           return VNET_API_ERROR_UNSPECIFIED;
1580         }
1581
1582       /* *INDENT-OFF* */
1583       pool_foreach (local, m->locals,
1584       ({
1585           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1586                             nat_fib_src_low);
1587           m_key.addr = local->addr;
1588           if (!out2in_only)
1589             {
1590               m_key.port = local->port;
1591               m_key.fib_index = local->fib_index;
1592               kv.key = m_key.as_u64;
1593               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1594                 {
1595                   nat_elog_err ("static_mapping_by_local key del failed");
1596                   return VNET_API_ERROR_UNSPECIFIED;
1597                 }
1598             }
1599
1600           if (sm->num_workers > 1)
1601             {
1602               ip4_header_t ip = {
1603                 .src_address = local->addr,
1604               };
1605               tsm = vec_elt_at_index (sm->per_thread_data,
1606                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1607             }
1608           else
1609             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1610
1611           /* Delete sessions */
1612           pool_foreach (s, tsm->sessions, {
1613             if (!(is_lb_session (s)))
1614               continue;
1615
1616             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1617                 (clib_net_to_host_u16 (s->in2out.port) != local->port))
1618               continue;
1619
1620             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1621             nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
1622           });
1623       }));
1624       /* *INDENT-ON* */
1625       if (m->affinity)
1626         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1627       pool_free (m->locals);
1628       vec_free (m->tag);
1629       vec_free (m->workers);
1630
1631       pool_put (sm->static_mappings, m);
1632     }
1633
1634   return 0;
1635 }
1636
1637 int
1638 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1639                                        ip4_address_t l_addr, u16 l_port,
1640                                        snat_protocol_t proto, u32 vrf_id,
1641                                        u8 probability, u8 is_add)
1642 {
1643   snat_main_t *sm = &snat_main;
1644   snat_static_mapping_t *m = 0;
1645   snat_session_key_t m_key;
1646   clib_bihash_kv_8_8_t kv, value;
1647   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1648   snat_main_per_thread_data_t *tsm;
1649   snat_session_t *s;
1650   u32 *locals = 0;
1651   uword *bitmap = 0;
1652   int i;
1653
1654   if (!sm->endpoint_dependent)
1655     return VNET_API_ERROR_FEATURE_DISABLED;
1656
1657   m_key.addr = e_addr;
1658   m_key.port = e_port;
1659   m_key.protocol = proto;
1660   m_key.fib_index = 0;
1661   kv.key = m_key.as_u64;
1662   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1663     m = pool_elt_at_index (sm->static_mappings, value.value);
1664
1665   if (!m)
1666     return VNET_API_ERROR_NO_SUCH_ENTRY;
1667
1668   if (!is_lb_static_mapping (m))
1669     return VNET_API_ERROR_INVALID_VALUE;
1670
1671   /* *INDENT-OFF* */
1672   pool_foreach (local, m->locals,
1673   ({
1674     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1675         (local->vrf_id == vrf_id))
1676       {
1677         match_local = local;
1678         break;
1679       }
1680   }));
1681   /* *INDENT-ON* */
1682
1683   if (is_add)
1684     {
1685       if (match_local)
1686         return VNET_API_ERROR_VALUE_EXIST;
1687
1688       pool_get (m->locals, local);
1689       clib_memset (local, 0, sizeof (*local));
1690       local->addr.as_u32 = l_addr.as_u32;
1691       local->port = l_port;
1692       local->probability = probability;
1693       local->vrf_id = vrf_id;
1694       local->fib_index =
1695         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1696                                            nat_fib_src_low);
1697
1698       if (!is_out2in_only_static_mapping (m))
1699         {
1700           m_key.addr = l_addr;
1701           m_key.port = l_port;
1702           m_key.fib_index = local->fib_index;
1703           kv.key = m_key.as_u64;
1704           kv.value = m - sm->static_mappings;
1705           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1706             nat_elog_err ("static_mapping_by_local key add failed");
1707         }
1708     }
1709   else
1710     {
1711       if (!match_local)
1712         return VNET_API_ERROR_NO_SUCH_ENTRY;
1713
1714       if (pool_elts (m->locals) < 3)
1715         return VNET_API_ERROR_UNSPECIFIED;
1716
1717       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1718                         nat_fib_src_low);
1719
1720       if (!is_out2in_only_static_mapping (m))
1721         {
1722           m_key.addr = l_addr;
1723           m_key.port = l_port;
1724           m_key.fib_index = match_local->fib_index;
1725           kv.key = m_key.as_u64;
1726           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1727             nat_elog_err ("static_mapping_by_local key del failed");
1728         }
1729
1730       if (sm->num_workers > 1)
1731         {
1732           ip4_header_t ip = {
1733             .src_address = local->addr,
1734           };
1735           tsm = vec_elt_at_index (sm->per_thread_data,
1736                                   sm->worker_in2out_cb (&ip, m->fib_index,
1737                                                         0));
1738         }
1739       else
1740         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1741
1742       /* Delete sessions */
1743       /* *INDENT-OFF* */
1744       pool_foreach (s, tsm->sessions, {
1745         if (!(is_lb_session (s)))
1746           continue;
1747
1748         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1749             (clib_net_to_host_u16 (s->in2out.port) != match_local->port))
1750           continue;
1751
1752         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1753         nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
1754       });
1755       /* *INDENT-ON* */
1756
1757       pool_put (m->locals, match_local);
1758     }
1759
1760   vec_free (m->workers);
1761
1762   /* *INDENT-OFF* */
1763   pool_foreach (local, m->locals,
1764   ({
1765     vec_add1 (locals, local - m->locals);
1766     if (sm->num_workers > 1)
1767       {
1768         ip4_header_t ip;
1769         ip.src_address.as_u32 = local->addr.as_u32,
1770         bitmap = clib_bitmap_set (bitmap,
1771                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1772                                   1);
1773       }
1774   }));
1775   /* *INDENT-ON* */
1776
1777   ASSERT (vec_len (locals) > 1);
1778
1779   local = pool_elt_at_index (m->locals, locals[0]);
1780   local->prefix = local->probability;
1781   for (i = 1; i < vec_len (locals); i++)
1782     {
1783       local = pool_elt_at_index (m->locals, locals[i]);
1784       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1785       local->prefix = local->probability + prev_local->prefix;
1786     }
1787
1788   /* Assign workers */
1789   if (sm->num_workers > 1)
1790     {
1791       /* *INDENT-OFF* */
1792       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1793       /* *INDENT-ON* */
1794     }
1795
1796   return 0;
1797 }
1798
1799 int
1800 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1801                   u8 twice_nat)
1802 {
1803   snat_address_t *a = 0;
1804   snat_session_t *ses;
1805   u32 *ses_to_be_removed = 0, *ses_index;
1806   snat_main_per_thread_data_t *tsm;
1807   snat_static_mapping_t *m;
1808   snat_interface_t *interface;
1809   int i;
1810   snat_address_t *addresses =
1811     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1812
1813   /* Find SNAT address */
1814   for (i = 0; i < vec_len (addresses); i++)
1815     {
1816       if (addresses[i].addr.as_u32 == addr.as_u32)
1817         {
1818           a = addresses + i;
1819           break;
1820         }
1821     }
1822   if (!a)
1823     return VNET_API_ERROR_NO_SUCH_ENTRY;
1824
1825   if (delete_sm)
1826     {
1827       /* *INDENT-OFF* */
1828       pool_foreach (m, sm->static_mappings,
1829       ({
1830           if (m->external_addr.as_u32 == addr.as_u32)
1831             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1832                                             m->local_port, m->external_port,
1833                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1834                                             m->proto, 0, m->twice_nat,
1835                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1836       }));
1837       /* *INDENT-ON* */
1838     }
1839   else
1840     {
1841       /* Check if address is used in some static mapping */
1842       if (is_snat_address_used_in_static_mapping (sm, addr))
1843         {
1844           nat_elog_notice ("address used in static mapping");
1845           return VNET_API_ERROR_UNSPECIFIED;
1846         }
1847     }
1848
1849   if (a->fib_index != ~0)
1850     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1851
1852   /* Delete sessions using address */
1853   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1854     {
1855       /* *INDENT-OFF* */
1856       vec_foreach (tsm, sm->per_thread_data)
1857         {
1858           pool_foreach (ses, tsm->sessions, ({
1859             if (ses->out2in.addr.as_u32 == addr.as_u32)
1860               {
1861                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1862                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1863               }
1864           }));
1865
1866           if (sm->endpoint_dependent){
1867               vec_foreach (ses_index, ses_to_be_removed)
1868                 {
1869                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1870                   nat44_ed_delete_session (sm, ses, tsm - sm->per_thread_data, 1);
1871                 }
1872           }else{
1873               vec_foreach (ses_index, ses_to_be_removed)
1874                 {
1875                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1876                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1877                 }
1878           }
1879
1880           vec_free (ses_to_be_removed);
1881         }
1882       /* *INDENT-ON* */
1883     }
1884
1885 #define _(N, i, n, s) \
1886   vec_free (a->busy_##n##_ports_per_thread);
1887   foreach_snat_protocol
1888 #undef _
1889     if (twice_nat)
1890     {
1891       vec_del1 (sm->twice_nat_addresses, i);
1892       return 0;
1893     }
1894   else
1895     vec_del1 (sm->addresses, i);
1896
1897   /* Delete external address from FIB */
1898   /* *INDENT-OFF* */
1899   pool_foreach (interface, sm->interfaces,
1900   ({
1901     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1902       continue;
1903
1904     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1905     break;
1906   }));
1907   pool_foreach (interface, sm->output_feature_interfaces,
1908   ({
1909     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1910       continue;
1911
1912     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1913     break;
1914   }));
1915   /* *INDENT-ON* */
1916
1917   return 0;
1918 }
1919
1920 int
1921 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1922 {
1923   snat_main_t *sm = &snat_main;
1924   snat_interface_t *i;
1925   const char *feature_name, *del_feature_name;
1926   snat_address_t *ap;
1927   snat_static_mapping_t *m;
1928   snat_det_map_t *dm;
1929   nat_outside_fib_t *outside_fib;
1930   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1931                                                        sw_if_index);
1932
1933   if (sm->out2in_dpo && !is_inside)
1934     return VNET_API_ERROR_UNSUPPORTED;
1935
1936   /* *INDENT-OFF* */
1937   pool_foreach (i, sm->output_feature_interfaces,
1938   ({
1939     if (i->sw_if_index == sw_if_index)
1940       return VNET_API_ERROR_VALUE_EXIST;
1941   }));
1942   /* *INDENT-ON* */
1943
1944   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1945     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1946   else
1947     {
1948       if (sm->num_workers > 1 && !sm->deterministic)
1949         feature_name =
1950           is_inside ? "nat44-in2out-worker-handoff" :
1951           "nat44-out2in-worker-handoff";
1952       else if (sm->deterministic)
1953         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1954       else if (sm->endpoint_dependent)
1955         {
1956           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1957         }
1958       else
1959         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1960     }
1961
1962   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1963     sm->fq_in2out_index =
1964       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1965
1966   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1967     sm->fq_out2in_index =
1968       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1969
1970   if (!is_inside)
1971     {
1972       /* *INDENT-OFF* */
1973       vec_foreach (outside_fib, sm->outside_fibs)
1974         {
1975           if (outside_fib->fib_index == fib_index)
1976             {
1977               if (is_del)
1978                 {
1979                   outside_fib->refcount--;
1980                   if (!outside_fib->refcount)
1981                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1982                 }
1983               else
1984                 outside_fib->refcount++;
1985               goto feature_set;
1986             }
1987         }
1988       /* *INDENT-ON* */
1989       if (!is_del)
1990         {
1991           vec_add2 (sm->outside_fibs, outside_fib, 1);
1992           outside_fib->refcount = 1;
1993           outside_fib->fib_index = fib_index;
1994         }
1995     }
1996 feature_set:
1997   /* *INDENT-OFF* */
1998   pool_foreach (i, sm->interfaces,
1999   ({
2000     if (i->sw_if_index == sw_if_index)
2001       {
2002         if (is_del)
2003           {
2004             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
2005               {
2006                 if (is_inside)
2007                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
2008                 else
2009                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
2010
2011                 if (sm->num_workers > 1 && !sm->deterministic)
2012                   {
2013                     del_feature_name = "nat44-handoff-classify";
2014                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2015                                                  "nat44-out2in-worker-handoff";
2016                   }
2017                 else if (sm->deterministic)
2018                   {
2019                     del_feature_name = "nat44-det-classify";
2020                     feature_name = !is_inside ?  "nat44-det-in2out" :
2021                                                  "nat44-det-out2in";
2022                   }
2023                 else if (sm->endpoint_dependent)
2024                   {
2025                     del_feature_name = "nat44-ed-classify";
2026                     feature_name = !is_inside ?  "nat-pre-in2out" :
2027                                                  "nat-pre-out2in";
2028                   }
2029                 else
2030                   {
2031                     del_feature_name = "nat44-classify";
2032                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2033                   }
2034
2035                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2036                 if (rv)
2037                   return rv;
2038                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2039                                              sw_if_index, 0, 0, 0);
2040                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2041                                              sw_if_index, 1, 0, 0);
2042                 if (!is_inside)
2043                   {
2044                     if (sm->endpoint_dependent)
2045                       vnet_feature_enable_disable ("ip4-local",
2046                                                    "nat44-ed-hairpinning",
2047                                                    sw_if_index, 1, 0, 0);
2048                     else if (!sm->deterministic)
2049                       vnet_feature_enable_disable ("ip4-local",
2050                                                    "nat44-hairpinning",
2051                                                    sw_if_index, 1, 0, 0);
2052                   }
2053               }
2054             else
2055               {
2056                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2057                 if (rv)
2058                   return rv;
2059                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2060                                              sw_if_index, 0, 0, 0);
2061                 pool_put (sm->interfaces, i);
2062                 if (is_inside)
2063                   {
2064                     if (sm->endpoint_dependent)
2065                       vnet_feature_enable_disable ("ip4-local",
2066                                                    "nat44-ed-hairpinning",
2067                                                    sw_if_index, 0, 0, 0);
2068                     else if (!sm->deterministic)
2069                       vnet_feature_enable_disable ("ip4-local",
2070                                                    "nat44-hairpinning",
2071                                                    sw_if_index, 0, 0, 0);
2072                   }
2073               }
2074           }
2075         else
2076           {
2077             if ((nat_interface_is_inside(i) && is_inside) ||
2078                 (nat_interface_is_outside(i) && !is_inside))
2079               return 0;
2080
2081             if (sm->num_workers > 1 && !sm->deterministic)
2082               {
2083                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2084                                                  "nat44-out2in-worker-handoff";
2085                 feature_name = "nat44-handoff-classify";
2086               }
2087             else if (sm->deterministic)
2088               {
2089                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
2090                                                  "nat44-det-out2in";
2091                 feature_name = "nat44-det-classify";
2092               }
2093             else if (sm->endpoint_dependent)
2094               {
2095                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2096                                                  "nat-pre-out2in";
2097
2098                 feature_name = "nat44-ed-classify";
2099               }
2100             else
2101               {
2102                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2103                 feature_name = "nat44-classify";
2104               }
2105
2106             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2107             if (rv)
2108               return rv;
2109             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2110                                          sw_if_index, 0, 0, 0);
2111             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2112                                          sw_if_index, 1, 0, 0);
2113             if (!is_inside)
2114               {
2115                 if (sm->endpoint_dependent)
2116                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2117                                                sw_if_index, 0, 0, 0);
2118                 else if (!sm->deterministic)
2119                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2120                                                sw_if_index, 0, 0, 0);
2121               }
2122             goto set_flags;
2123           }
2124
2125         goto fib;
2126       }
2127   }));
2128   /* *INDENT-ON* */
2129
2130   if (is_del)
2131     return VNET_API_ERROR_NO_SUCH_ENTRY;
2132
2133   pool_get (sm->interfaces, i);
2134   i->sw_if_index = sw_if_index;
2135   i->flags = 0;
2136   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2137                                0);
2138
2139   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2140   if (rv)
2141     return rv;
2142
2143   if (is_inside && !sm->out2in_dpo)
2144     {
2145       if (sm->endpoint_dependent)
2146         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2147                                      sw_if_index, 1, 0, 0);
2148       else if (!sm->deterministic)
2149         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2150                                      sw_if_index, 1, 0, 0);
2151     }
2152
2153 set_flags:
2154   if (is_inside)
2155     {
2156       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2157       return 0;
2158     }
2159   else
2160     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2161
2162   /* Add/delete external addresses to FIB */
2163 fib:
2164   /* *INDENT-OFF* */
2165   vec_foreach (ap, sm->addresses)
2166     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2167
2168   pool_foreach (m, sm->static_mappings,
2169   ({
2170     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2171       continue;
2172
2173     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2174   }));
2175
2176   pool_foreach (dm, sm->det_maps,
2177   ({
2178     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2179   }));
2180   /* *INDENT-ON* */
2181
2182   return 0;
2183 }
2184
2185 int
2186 snat_interface_add_del_output_feature (u32 sw_if_index,
2187                                        u8 is_inside, int is_del)
2188 {
2189   snat_main_t *sm = &snat_main;
2190   snat_interface_t *i;
2191   snat_address_t *ap;
2192   snat_static_mapping_t *m;
2193   nat_outside_fib_t *outside_fib;
2194   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2195                                                        sw_if_index);
2196
2197
2198   if (sm->deterministic ||
2199       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2200     return VNET_API_ERROR_UNSUPPORTED;
2201
2202   /* *INDENT-OFF* */
2203   pool_foreach (i, sm->interfaces,
2204   ({
2205     if (i->sw_if_index == sw_if_index)
2206       return VNET_API_ERROR_VALUE_EXIST;
2207   }));
2208   /* *INDENT-ON* */
2209
2210   if (!is_inside)
2211     {
2212       /* *INDENT-OFF* */
2213       vec_foreach (outside_fib, sm->outside_fibs)
2214         {
2215           if (outside_fib->fib_index == fib_index)
2216             {
2217               if (is_del)
2218                 {
2219                   outside_fib->refcount--;
2220                   if (!outside_fib->refcount)
2221                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2222                 }
2223               else
2224                 outside_fib->refcount++;
2225               goto feature_set;
2226             }
2227         }
2228       /* *INDENT-ON* */
2229       if (!is_del)
2230         {
2231           vec_add2 (sm->outside_fibs, outside_fib, 1);
2232           outside_fib->refcount = 1;
2233           outside_fib->fib_index = fib_index;
2234         }
2235     }
2236
2237 feature_set:
2238   if (is_inside)
2239     {
2240       if (sm->endpoint_dependent)
2241         {
2242           int rv =
2243             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2244           if (rv)
2245             return rv;
2246           rv =
2247             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2248                                                             !is_del);
2249           if (rv)
2250             return rv;
2251           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2252                                        sw_if_index, !is_del, 0, 0);
2253           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2254                                        sw_if_index, !is_del, 0, 0);
2255         }
2256       else
2257         {
2258           int rv =
2259             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2260           if (rv)
2261             return rv;
2262           rv =
2263             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2264                                                             !is_del);
2265           if (rv)
2266             return rv;
2267           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2268                                        sw_if_index, !is_del, 0, 0);
2269           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2270                                        sw_if_index, !is_del, 0, 0);
2271         }
2272       goto fq;
2273     }
2274
2275   if (sm->num_workers > 1)
2276     {
2277       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2278       if (rv)
2279         return rv;
2280       rv =
2281         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2282       if (rv)
2283         return rv;
2284       vnet_feature_enable_disable ("ip4-unicast",
2285                                    "nat44-out2in-worker-handoff",
2286                                    sw_if_index, !is_del, 0, 0);
2287       vnet_feature_enable_disable ("ip4-output",
2288                                    "nat44-in2out-output-worker-handoff",
2289                                    sw_if_index, !is_del, 0, 0);
2290     }
2291   else
2292     {
2293       if (sm->endpoint_dependent)
2294         {
2295           int rv =
2296             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2297           if (rv)
2298             return rv;
2299           rv =
2300             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2301                                                             !is_del);
2302           if (rv)
2303             return rv;
2304           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2305                                        sw_if_index, !is_del, 0, 0);
2306           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2307                                        sw_if_index, !is_del, 0, 0);
2308         }
2309       else
2310         {
2311           int rv =
2312             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2313           if (rv)
2314             return rv;
2315           rv =
2316             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2317                                                             !is_del);
2318           if (rv)
2319             return rv;
2320           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2321                                        sw_if_index, !is_del, 0, 0);
2322           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2323                                        sw_if_index, !is_del, 0, 0);
2324         }
2325     }
2326
2327 fq:
2328   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2329     sm->fq_in2out_output_index =
2330       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2331
2332   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2333     sm->fq_out2in_index =
2334       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2335
2336   /* *INDENT-OFF* */
2337   pool_foreach (i, sm->output_feature_interfaces,
2338   ({
2339     if (i->sw_if_index == sw_if_index)
2340       {
2341         if (is_del)
2342           pool_put (sm->output_feature_interfaces, i);
2343         else
2344           return VNET_API_ERROR_VALUE_EXIST;
2345
2346         goto fib;
2347       }
2348   }));
2349   /* *INDENT-ON* */
2350
2351   if (is_del)
2352     return VNET_API_ERROR_NO_SUCH_ENTRY;
2353
2354   pool_get (sm->output_feature_interfaces, i);
2355   i->sw_if_index = sw_if_index;
2356   i->flags = 0;
2357   if (is_inside)
2358     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2359   else
2360     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2361
2362   /* Add/delete external addresses to FIB */
2363 fib:
2364   if (is_inside)
2365     return 0;
2366
2367   /* *INDENT-OFF* */
2368   vec_foreach (ap, sm->addresses)
2369     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2370
2371   pool_foreach (m, sm->static_mappings,
2372   ({
2373     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2374       continue;
2375
2376     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2377   }));
2378   /* *INDENT-ON* */
2379
2380   return 0;
2381 }
2382
2383 int
2384 snat_set_workers (uword * bitmap)
2385 {
2386   snat_main_t *sm = &snat_main;
2387   int i, j = 0;
2388
2389   if (sm->num_workers < 2)
2390     return VNET_API_ERROR_FEATURE_DISABLED;
2391
2392   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2393     return VNET_API_ERROR_INVALID_WORKER;
2394
2395   vec_free (sm->workers);
2396   /* *INDENT-OFF* */
2397   clib_bitmap_foreach (i, bitmap,
2398     ({
2399       vec_add1(sm->workers, i);
2400       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2401       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2402       j++;
2403     }));
2404   /* *INDENT-ON* */
2405
2406   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2407   sm->num_snat_thread = _vec_len (sm->workers);
2408
2409   return 0;
2410 }
2411
2412 static void
2413 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2414                          u32 old_fib_index)
2415 {
2416   snat_main_t *sm = &snat_main;
2417   nat_outside_fib_t *outside_fib;
2418   snat_interface_t *i;
2419   u8 is_add = 1;
2420   u8 match = 0;
2421
2422   if (new_fib_index == old_fib_index)
2423     return;
2424
2425   if (!vec_len (sm->outside_fibs))
2426     return;
2427
2428   /* *INDENT-OFF* */
2429   pool_foreach (i, sm->interfaces,
2430     ({
2431       if (i->sw_if_index == sw_if_index)
2432         {
2433           if (!(nat_interface_is_outside (i)))
2434             return;
2435           match = 1;
2436         }
2437     }));
2438
2439   pool_foreach (i, sm->output_feature_interfaces,
2440     ({
2441       if (i->sw_if_index == sw_if_index)
2442         {
2443           if (!(nat_interface_is_outside (i)))
2444             return;
2445           match = 1;
2446         }
2447     }));
2448   /* *INDENT-ON* */
2449
2450   if (!match)
2451     return;
2452
2453   vec_foreach (outside_fib, sm->outside_fibs)
2454   {
2455     if (outside_fib->fib_index == old_fib_index)
2456       {
2457         outside_fib->refcount--;
2458         if (!outside_fib->refcount)
2459           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2460         break;
2461       }
2462   }
2463
2464   vec_foreach (outside_fib, sm->outside_fibs)
2465   {
2466     if (outside_fib->fib_index == new_fib_index)
2467       {
2468         outside_fib->refcount++;
2469         is_add = 0;
2470         break;
2471       }
2472   }
2473
2474   if (is_add)
2475     {
2476       vec_add2 (sm->outside_fibs, outside_fib, 1);
2477       outside_fib->refcount = 1;
2478       outside_fib->fib_index = new_fib_index;
2479     }
2480 }
2481
2482 static void
2483 snat_ip4_table_bind (ip4_main_t * im,
2484                      uword opaque,
2485                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2486 {
2487   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2488 }
2489
2490 static void
2491 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2492                                        uword opaque,
2493                                        u32 sw_if_index,
2494                                        ip4_address_t * address,
2495                                        u32 address_length,
2496                                        u32 if_address_index, u32 is_delete);
2497
2498 static void
2499 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2500                                  uword opaque,
2501                                  u32 sw_if_index,
2502                                  ip4_address_t * address,
2503                                  u32 address_length,
2504                                  u32 if_address_index, u32 is_delete);
2505
2506 static int
2507 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2508                                  u32 fib_index,
2509                                  u32 thread_index,
2510                                  snat_session_key_t * k,
2511                                  u16 port_per_thread, u32 snat_thread_index);
2512
2513 void
2514 test_ed_make_split ()
2515 {
2516   ip4_address_t l_addr;
2517   l_addr.as_u8[0] = 1;
2518   l_addr.as_u8[1] = 1;
2519   l_addr.as_u8[2] = 1;
2520   l_addr.as_u8[3] = 1;
2521   ip4_address_t r_addr;
2522   r_addr.as_u8[0] = 2;
2523   r_addr.as_u8[1] = 2;
2524   r_addr.as_u8[2] = 2;
2525   r_addr.as_u8[3] = 2;
2526   u16 l_port = 40001;
2527   u16 r_port = 40301;
2528   u8 proto = 9;
2529   u32 fib_index = 9000001;
2530   u64 value = ~0ULL;
2531   clib_bihash_kv_16_8_t kv;
2532   make_ed_kv (&l_addr, &r_addr, proto, fib_index, l_port, r_port, value, &kv);
2533   ip4_address_t l_addr2;
2534   ip4_address_t r_addr2;
2535   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2536   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2537   u16 l_port2 = 0;
2538   u16 r_port2 = 0;
2539   u8 proto2 = 0;
2540   u32 fib_index2 = 0;
2541   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2542                &r_port2);
2543   u64 value2 = kv.value;
2544   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2545   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2546   ASSERT (l_port == l_port2);
2547   ASSERT (r_port == r_port2);
2548   ASSERT (proto == proto2);
2549   ASSERT (fib_index == fib_index2);
2550   ASSERT (value == value2);
2551 }
2552
2553 always_inline vlib_main_t *
2554 nat_get_vlib_main (u32 thread_index)
2555 {
2556   vlib_main_t *vm;
2557   vm = vlib_mains[thread_index];
2558   ASSERT (vm);
2559   return vm;
2560 }
2561
2562 static clib_error_t *
2563 snat_init (vlib_main_t * vm)
2564 {
2565   snat_main_per_thread_data_t *tsm;
2566
2567   snat_main_t *sm = &snat_main;
2568   clib_error_t *error = 0;
2569   ip4_main_t *im = &ip4_main;
2570   ip_lookup_main_t *lm = &im->lookup_main;
2571   uword *p;
2572   vlib_thread_registration_t *tr;
2573   vlib_thread_main_t *tm = vlib_get_thread_main ();
2574   uword *bitmap = 0;
2575   u32 i;
2576   ip4_add_del_interface_address_callback_t cb4;
2577   vlib_node_t *node;
2578
2579   sm->vnet_main = vnet_get_main ();
2580   sm->ip4_main = im;
2581   sm->ip4_lookup_main = lm;
2582   sm->api_main = vlibapi_get_main ();
2583   sm->first_worker_index = 0;
2584   sm->num_workers = 0;
2585   sm->num_snat_thread = 1;
2586   sm->workers = 0;
2587   sm->port_per_thread = 0xffff - 1024;
2588   sm->fq_in2out_index = ~0;
2589   sm->fq_in2out_output_index = ~0;
2590   sm->fq_out2in_index = ~0;
2591
2592   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2593   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2594   sm->forwarding_enabled = 0;
2595   sm->log_class = vlib_log_register_class ("nat", 0);
2596   sm->log_level = SNAT_LOG_ERROR;
2597   sm->mss_clamping = 0;
2598
2599   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2600   sm->error_node_index = node->index;
2601
2602   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2603   sm->pre_in2out_node_index = node->index;
2604   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2605   sm->pre_out2in_node_index = node->index;
2606
2607   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2608   sm->pre_in2out_node_index = node->index;
2609
2610   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2611   sm->pre_out2in_node_index = node->index;
2612
2613   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2614   sm->in2out_node_index = node->index;
2615   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2616   sm->in2out_output_node_index = node->index;
2617   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2618   sm->in2out_fast_node_index = node->index;
2619   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2620   sm->in2out_slowpath_node_index = node->index;
2621   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2622   sm->in2out_slowpath_output_node_index = node->index;
2623
2624   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2625   sm->ed_in2out_node_index = node->index;
2626   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2627   sm->ed_in2out_slowpath_node_index = node->index;
2628
2629   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2630   sm->out2in_node_index = node->index;
2631   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2632   sm->out2in_fast_node_index = node->index;
2633
2634   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2635   sm->ed_out2in_node_index = node->index;
2636   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2637   sm->ed_out2in_slowpath_node_index = node->index;
2638
2639   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2640   sm->det_in2out_node_index = node->index;
2641   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2642   sm->det_out2in_node_index = node->index;
2643
2644   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2645   sm->hairpinning_node_index = node->index;
2646   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2647   sm->hairpin_dst_node_index = node->index;
2648   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2649   sm->hairpin_src_node_index = node->index;
2650   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2651   sm->ed_hairpinning_node_index = node->index;
2652   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2653   sm->ed_hairpin_dst_node_index = node->index;
2654   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2655   sm->ed_hairpin_src_node_index = node->index;
2656
2657   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2658   if (p)
2659     {
2660       tr = (vlib_thread_registration_t *) p[0];
2661       if (tr)
2662         {
2663           sm->num_workers = tr->count;
2664           sm->first_worker_index = tr->first_index;
2665         }
2666     }
2667
2668   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2669
2670   /* Use all available workers by default */
2671   if (sm->num_workers > 1)
2672     {
2673       for (i = 0; i < sm->num_workers; i++)
2674         bitmap = clib_bitmap_set (bitmap, i, 1);
2675       // sets thread indexes for workes
2676       snat_set_workers (bitmap);
2677       clib_bitmap_free (bitmap);
2678     }
2679   else
2680     {
2681       sm->per_thread_data[0].snat_thread_index = 0;
2682     }
2683
2684   vec_foreach (tsm, sm->per_thread_data)
2685   {
2686     tsm->vlib_main = nat_get_vlib_main (tsm->thread_index);
2687   }
2688
2689   error = snat_api_init (vm, sm);
2690   if (error)
2691     return error;
2692
2693   /* Set up the interface address add/del callback */
2694   cb4.function = snat_ip4_add_del_interface_address_cb;
2695   cb4.function_opaque = 0;
2696
2697   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2698
2699   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2700   cb4.function_opaque = 0;
2701
2702   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2703
2704   nat_dpo_module_init ();
2705
2706   /* Init counters */
2707   sm->total_users.name = "total-users";
2708   sm->total_users.stat_segment_name = "/nat44/total-users";
2709   vlib_validate_simple_counter (&sm->total_users, 0);
2710   vlib_zero_simple_counter (&sm->total_users, 0);
2711   sm->total_sessions.name = "total-sessions";
2712   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2713   vlib_validate_simple_counter (&sm->total_sessions, 0);
2714   vlib_zero_simple_counter (&sm->total_sessions, 0);
2715
2716   /* Init IPFIX logging */
2717   snat_ipfix_logging_init (vm);
2718
2719   /* Init NAT64 */
2720   error = nat64_init (vm);
2721   if (error)
2722     return error;
2723
2724   nat66_init (vm);
2725
2726   ip4_table_bind_callback_t cbt4 = {
2727     .function = snat_ip4_table_bind,
2728   };
2729   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2730
2731   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2732                                         FIB_SOURCE_PRIORITY_HI,
2733                                         FIB_SOURCE_BH_SIMPLE);
2734   nat_fib_src_low = fib_source_allocate ("nat-low",
2735                                          FIB_SOURCE_PRIORITY_LOW,
2736                                          FIB_SOURCE_BH_SIMPLE);
2737
2738   test_ed_make_split ();
2739   return error;
2740 }
2741
2742 VLIB_INIT_FUNCTION (snat_init);
2743
2744 void
2745 snat_free_outside_address_and_port (snat_address_t * addresses,
2746                                     u32 thread_index, snat_session_key_t * k)
2747 {
2748   snat_address_t *a;
2749   u32 address_index;
2750   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2751
2752   for (address_index = 0; address_index < vec_len (addresses);
2753        address_index++)
2754     {
2755       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2756         break;
2757     }
2758
2759   ASSERT (address_index < vec_len (addresses));
2760
2761   a = addresses + address_index;
2762
2763   switch (k->protocol)
2764     {
2765 #define _(N, i, n, s) \
2766     case SNAT_PROTOCOL_##N: \
2767       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2768       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2769       a->busy_##n##_ports--; \
2770       a->busy_##n##_ports_per_thread[thread_index]--; \
2771       break;
2772       foreach_snat_protocol
2773 #undef _
2774     default:
2775       nat_elog_info ("unknown protocol");
2776       return;
2777     }
2778 }
2779
2780 static int
2781 nat_set_outside_address_and_port (snat_address_t * addresses,
2782                                   u32 thread_index, snat_session_key_t * k)
2783 {
2784   snat_address_t *a = 0;
2785   u32 address_index;
2786   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2787
2788   for (address_index = 0; address_index < vec_len (addresses);
2789        address_index++)
2790     {
2791       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2792         continue;
2793
2794       a = addresses + address_index;
2795       switch (k->protocol)
2796         {
2797 #define _(N, j, n, s) \
2798         case SNAT_PROTOCOL_##N: \
2799           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2800             return VNET_API_ERROR_INSTANCE_IN_USE; \
2801           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2802           a->busy_##n##_ports_per_thread[thread_index]++; \
2803           a->busy_##n##_ports++; \
2804           return 0;
2805           foreach_snat_protocol
2806 #undef _
2807         default:
2808           nat_elog_info ("unknown protocol");
2809           return 1;
2810         }
2811     }
2812
2813   return VNET_API_ERROR_NO_SUCH_ENTRY;
2814 }
2815
2816 int
2817 snat_static_mapping_match (snat_main_t * sm,
2818                            snat_session_key_t match,
2819                            snat_session_key_t * mapping,
2820                            u8 by_external,
2821                            u8 * is_addr_only,
2822                            twice_nat_type_t * twice_nat,
2823                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2824                            u8 * is_identity_nat)
2825 {
2826   clib_bihash_kv_8_8_t kv, value;
2827   snat_static_mapping_t *m;
2828   snat_session_key_t m_key;
2829   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2830   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2831   u8 backend_index;
2832   nat44_lb_addr_port_t *local;
2833
2834   m_key.fib_index = match.fib_index;
2835   if (by_external)
2836     {
2837       mapping_hash = &sm->static_mapping_by_external;
2838       m_key.fib_index = 0;
2839     }
2840
2841   m_key.addr = match.addr;
2842   m_key.port = clib_net_to_host_u16 (match.port);
2843   m_key.protocol = match.protocol;
2844
2845   kv.key = m_key.as_u64;
2846
2847   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2848     {
2849       /* Try address only mapping */
2850       m_key.port = 0;
2851       m_key.protocol = 0;
2852       kv.key = m_key.as_u64;
2853       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2854         return 1;
2855     }
2856
2857   m = pool_elt_at_index (sm->static_mappings, value.value);
2858
2859   if (by_external)
2860     {
2861       if (is_lb_static_mapping (m))
2862         {
2863           if (PREDICT_FALSE (lb != 0))
2864             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2865           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2866                                                           match.addr,
2867                                                           match.protocol,
2868                                                           match.port,
2869                                                           &backend_index))
2870             {
2871               local = pool_elt_at_index (m->locals, backend_index);
2872               mapping->addr = local->addr;
2873               mapping->port = clib_host_to_net_u16 (local->port);
2874               mapping->fib_index = local->fib_index;
2875               goto end;
2876             }
2877           // pick locals matching this worker
2878           if (PREDICT_FALSE (sm->num_workers > 1))
2879             {
2880               u32 thread_index = vlib_get_thread_index ();
2881               /* *INDENT-OFF* */
2882               pool_foreach_index (i, m->locals,
2883               ({
2884                 local = pool_elt_at_index (m->locals, i);
2885
2886                 ip4_header_t ip = {
2887                   .src_address = local->addr,
2888                 };
2889
2890                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2891                     thread_index)
2892                   {
2893                     vec_add1 (tmp, i);
2894                   }
2895               }));
2896               /* *INDENT-ON* */
2897               ASSERT (vec_len (tmp) != 0);
2898             }
2899           else
2900             {
2901               /* *INDENT-OFF* */
2902               pool_foreach_index (i, m->locals,
2903               ({
2904                 vec_add1 (tmp, i);
2905               }));
2906               /* *INDENT-ON* */
2907             }
2908           hi = vec_len (tmp) - 1;
2909           local = pool_elt_at_index (m->locals, tmp[hi]);
2910           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2911           while (lo < hi)
2912             {
2913               mid = ((hi - lo) >> 1) + lo;
2914               local = pool_elt_at_index (m->locals, tmp[mid]);
2915               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2916             }
2917           local = pool_elt_at_index (m->locals, tmp[lo]);
2918           if (!(local->prefix >= rand))
2919             return 1;
2920           mapping->addr = local->addr;
2921           mapping->port = clib_host_to_net_u16 (local->port);
2922           mapping->fib_index = local->fib_index;
2923           if (m->affinity)
2924             {
2925               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2926                                                 match.protocol, match.port,
2927                                                 tmp[lo], m->affinity,
2928                                                 m->affinity_per_service_list_head_index))
2929                 nat_elog_info ("create affinity record failed");
2930             }
2931           vec_free (tmp);
2932         }
2933       else
2934         {
2935           if (PREDICT_FALSE (lb != 0))
2936             *lb = NO_LB_NAT;
2937           mapping->fib_index = m->fib_index;
2938           mapping->addr = m->local_addr;
2939           /* Address only mapping doesn't change port */
2940           mapping->port = is_addr_only_static_mapping (m) ? match.port
2941             : clib_host_to_net_u16 (m->local_port);
2942         }
2943       mapping->protocol = m->proto;
2944     }
2945   else
2946     {
2947       mapping->addr = m->external_addr;
2948       /* Address only mapping doesn't change port */
2949       mapping->port = is_addr_only_static_mapping (m) ? match.port
2950         : clib_host_to_net_u16 (m->external_port);
2951       mapping->fib_index = sm->outside_fib_index;
2952     }
2953
2954 end:
2955   if (PREDICT_FALSE (is_addr_only != 0))
2956     *is_addr_only = is_addr_only_static_mapping (m);
2957
2958   if (PREDICT_FALSE (twice_nat != 0))
2959     *twice_nat = m->twice_nat;
2960
2961   if (PREDICT_FALSE (is_identity_nat != 0))
2962     *is_identity_nat = is_identity_static_mapping (m);
2963
2964   return 0;
2965 }
2966
2967 static_always_inline u16
2968 snat_random_port (u16 min, u16 max)
2969 {
2970   snat_main_t *sm = &snat_main;
2971   return min + random_u32 (&sm->random_seed) /
2972     (random_u32_max () / (max - min + 1) + 1);
2973 }
2974
2975 int
2976 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2977                                      u32 fib_index,
2978                                      u32 thread_index,
2979                                      snat_session_key_t * k,
2980                                      u16 port_per_thread,
2981                                      u32 snat_thread_index)
2982 {
2983   snat_main_t *sm = &snat_main;
2984
2985   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2986                                   port_per_thread, snat_thread_index);
2987 }
2988
2989 static int
2990 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2991                                  u32 fib_index,
2992                                  u32 thread_index,
2993                                  snat_session_key_t * k,
2994                                  u16 port_per_thread, u32 snat_thread_index)
2995 {
2996   int i;
2997   snat_address_t *a, *ga = 0;
2998   u32 portnum;
2999
3000   for (i = 0; i < vec_len (addresses); i++)
3001     {
3002       a = addresses + i;
3003       switch (k->protocol)
3004         {
3005 #define _(N, j, n, s) \
3006         case SNAT_PROTOCOL_##N: \
3007           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
3008             { \
3009               if (a->fib_index == fib_index) \
3010                 { \
3011                   while (1) \
3012                     { \
3013                       portnum = (port_per_thread * \
3014                         snat_thread_index) + \
3015                         snat_random_port(1, port_per_thread) + 1024; \
3016                       if (a->busy_##n##_port_refcounts[portnum]) \
3017                         continue; \
3018                       --a->busy_##n##_port_refcounts[portnum]; \
3019                       a->busy_##n##_ports_per_thread[thread_index]++; \
3020                       a->busy_##n##_ports++; \
3021                       k->addr = a->addr; \
3022                       k->port = clib_host_to_net_u16(portnum); \
3023                       return 0; \
3024                     } \
3025                 } \
3026               else if (a->fib_index == ~0) \
3027                 { \
3028                   ga = a; \
3029                 } \
3030             } \
3031           break;
3032           foreach_snat_protocol
3033 #undef _
3034         default:
3035           nat_elog_info ("unknown protocol");
3036           return 1;
3037         }
3038
3039     }
3040
3041   if (ga)
3042     {
3043       a = ga;
3044       switch (k->protocol)
3045         {
3046 #define _(N, j, n, s) \
3047         case SNAT_PROTOCOL_##N: \
3048           while (1) \
3049             { \
3050               portnum = (port_per_thread * \
3051                 snat_thread_index) + \
3052                 snat_random_port(1, port_per_thread) + 1024; \
3053               if (a->busy_##n##_port_refcounts[portnum]) \
3054                 continue; \
3055               ++a->busy_##n##_port_refcounts[portnum]; \
3056               a->busy_##n##_ports_per_thread[thread_index]++; \
3057               a->busy_##n##_ports++; \
3058               k->addr = a->addr; \
3059               k->port = clib_host_to_net_u16(portnum); \
3060               return 0; \
3061             }
3062           break;
3063           foreach_snat_protocol
3064 #undef _
3065         default:
3066           nat_elog_info ("unknown protocol");
3067           return 1;
3068         }
3069     }
3070
3071   /* Totally out of translations to use... */
3072   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3073   return 1;
3074 }
3075
3076 static int
3077 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
3078                               u32 fib_index,
3079                               u32 thread_index,
3080                               snat_session_key_t * k,
3081                               u16 port_per_thread, u32 snat_thread_index)
3082 {
3083   snat_main_t *sm = &snat_main;
3084   snat_address_t *a = addresses;
3085   u16 m, ports, portnum, A, j;
3086   m = 16 - (sm->psid_offset + sm->psid_length);
3087   ports = (1 << (16 - sm->psid_length)) - (1 << m);
3088
3089   if (!vec_len (addresses))
3090     goto exhausted;
3091
3092   switch (k->protocol)
3093     {
3094 #define _(N, i, n, s) \
3095     case SNAT_PROTOCOL_##N: \
3096       if (a->busy_##n##_ports < ports) \
3097         { \
3098           while (1) \
3099             { \
3100               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3101               j = snat_random_port(0, pow2_mask(m)); \
3102               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3103               if (a->busy_##n##_port_refcounts[portnum]) \
3104                 continue; \
3105               ++a->busy_##n##_port_refcounts[portnum]; \
3106               a->busy_##n##_ports++; \
3107               k->addr = a->addr; \
3108               k->port = clib_host_to_net_u16 (portnum); \
3109               return 0; \
3110             } \
3111         } \
3112       break;
3113       foreach_snat_protocol
3114 #undef _
3115     default:
3116       nat_elog_info ("unknown protocol");
3117       return 1;
3118     }
3119
3120 exhausted:
3121   /* Totally out of translations to use... */
3122   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3123   return 1;
3124 }
3125
3126 static int
3127 nat_alloc_addr_and_port_range (snat_address_t * addresses,
3128                                u32 fib_index,
3129                                u32 thread_index,
3130                                snat_session_key_t * k,
3131                                u16 port_per_thread, u32 snat_thread_index)
3132 {
3133   snat_main_t *sm = &snat_main;
3134   snat_address_t *a = addresses;
3135   u16 portnum, ports;
3136
3137   ports = sm->end_port - sm->start_port + 1;
3138
3139   if (!vec_len (addresses))
3140     goto exhausted;
3141
3142   switch (k->protocol)
3143     {
3144 #define _(N, i, n, s) \
3145     case SNAT_PROTOCOL_##N: \
3146       if (a->busy_##n##_ports < ports) \
3147         { \
3148           while (1) \
3149             { \
3150               portnum = snat_random_port(sm->start_port, sm->end_port); \
3151               if (a->busy_##n##_port_refcounts[portnum]) \
3152                 continue; \
3153               ++a->busy_##n##_port_refcounts[portnum]; \
3154               a->busy_##n##_ports++; \
3155               k->addr = a->addr; \
3156               k->port = clib_host_to_net_u16 (portnum); \
3157               return 0; \
3158             } \
3159         } \
3160       break;
3161       foreach_snat_protocol
3162 #undef _
3163     default:
3164       nat_elog_info ("unknown protocol");
3165       return 1;
3166     }
3167
3168 exhausted:
3169   /* Totally out of translations to use... */
3170   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3171   return 1;
3172 }
3173
3174 void
3175 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3176 {
3177   dpo_id_t dpo_v4 = DPO_INVALID;
3178   fib_prefix_t pfx = {
3179     .fp_proto = FIB_PROTOCOL_IP4,
3180     .fp_len = 32,
3181     .fp_addr.ip4.as_u32 = addr.as_u32,
3182   };
3183
3184   if (is_add)
3185     {
3186       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3187       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3188                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3189       dpo_reset (&dpo_v4);
3190     }
3191   else
3192     {
3193       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3194     }
3195 }
3196
3197 u8 *
3198 format_session_kvp (u8 * s, va_list * args)
3199 {
3200   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3201   snat_session_key_t k;
3202
3203   k.as_u64 = v->key;
3204
3205   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
3206
3207   return s;
3208 }
3209
3210 u8 *
3211 format_static_mapping_kvp (u8 * s, va_list * args)
3212 {
3213   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3214   snat_session_key_t k;
3215
3216   k.as_u64 = v->key;
3217
3218   s = format (s, "%U static-mapping-index %llu",
3219               format_static_mapping_key, &k, v->value);
3220
3221   return s;
3222 }
3223
3224 u8 *
3225 format_user_kvp (u8 * s, va_list * args)
3226 {
3227   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3228   snat_user_key_t k;
3229
3230   k.as_u64 = v->key;
3231
3232   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3233               k.fib_index, v->value);
3234
3235   return s;
3236 }
3237
3238 u8 *
3239 format_ed_session_kvp (u8 * s, va_list * args)
3240 {
3241   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3242
3243   u8 proto;
3244   u16 r_port, l_port;
3245   ip4_address_t l_addr, r_addr;
3246   u32 fib_index;
3247
3248   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
3249   s =
3250     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3251             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
3252             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
3253             format_ip_protocol, proto, fib_index, v->value);
3254
3255   return s;
3256 }
3257
3258 static u32
3259 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3260                            u8 is_output)
3261 {
3262   snat_main_t *sm = &snat_main;
3263   u32 next_worker_index = 0;
3264   u32 hash;
3265
3266   next_worker_index = sm->first_worker_index;
3267   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3268     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3269
3270   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3271     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3272   else
3273     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3274
3275   return next_worker_index;
3276 }
3277
3278 static u32
3279 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3280                            u32 rx_fib_index0, u8 is_output)
3281 {
3282   snat_main_t *sm = &snat_main;
3283   udp_header_t *udp;
3284   u16 port;
3285   snat_session_key_t m_key;
3286   clib_bihash_kv_8_8_t kv, value;
3287   snat_static_mapping_t *m;
3288   u32 proto;
3289   u32 next_worker_index = 0;
3290
3291   /* first try static mappings without port */
3292   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3293     {
3294       m_key.addr = ip0->dst_address;
3295       m_key.port = 0;
3296       m_key.protocol = 0;
3297       m_key.fib_index = rx_fib_index0;
3298       kv.key = m_key.as_u64;
3299       if (!clib_bihash_search_8_8
3300           (&sm->static_mapping_by_external, &kv, &value))
3301         {
3302           m = pool_elt_at_index (sm->static_mappings, value.value);
3303           return m->workers[0];
3304         }
3305     }
3306
3307   proto = ip_proto_to_snat_proto (ip0->protocol);
3308   udp = ip4_next_header (ip0);
3309   port = udp->dst_port;
3310
3311   /* unknown protocol */
3312   if (PREDICT_FALSE (proto == ~0))
3313     {
3314       /* use current thread */
3315       return vlib_get_thread_index ();
3316     }
3317
3318   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3319     {
3320       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3321       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3322       if (!icmp_type_is_error_message
3323           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3324         port = vnet_buffer (b)->ip.reass.l4_src_port;
3325       else
3326         {
3327           /* if error message, then it's not fragmented and we can access it */
3328           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3329           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3330           void *l4_header = ip4_next_header (inner_ip);
3331           switch (proto)
3332             {
3333             case SNAT_PROTOCOL_ICMP:
3334               icmp = (icmp46_header_t *) l4_header;
3335               echo = (icmp_echo_header_t *) (icmp + 1);
3336               port = echo->identifier;
3337               break;
3338             case SNAT_PROTOCOL_UDP:
3339             case SNAT_PROTOCOL_TCP:
3340               port = ((tcp_udp_header_t *) l4_header)->src_port;
3341               break;
3342             default:
3343               return vlib_get_thread_index ();
3344             }
3345         }
3346     }
3347
3348   /* try static mappings with port */
3349   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3350     {
3351       m_key.addr = ip0->dst_address;
3352       m_key.port = clib_net_to_host_u16 (port);
3353       m_key.protocol = proto;
3354       m_key.fib_index = rx_fib_index0;
3355       kv.key = m_key.as_u64;
3356       if (!clib_bihash_search_8_8
3357           (&sm->static_mapping_by_external, &kv, &value))
3358         {
3359           m = pool_elt_at_index (sm->static_mappings, value.value);
3360           return m->workers[0];
3361         }
3362     }
3363
3364   /* worker by outside port */
3365   next_worker_index = sm->first_worker_index;
3366   next_worker_index +=
3367     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3368   return next_worker_index;
3369 }
3370
3371 static u32
3372 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3373                                u8 is_output)
3374 {
3375   snat_main_t *sm = &snat_main;
3376   u32 next_worker_index = sm->first_worker_index;
3377   u32 hash;
3378
3379   clib_bihash_kv_16_8_t kv16, value16;
3380   snat_main_per_thread_data_t *tsm;
3381   udp_header_t *udp;
3382
3383   if (PREDICT_FALSE (is_output))
3384     {
3385       u32 fib_index = sm->outside_fib_index;
3386       nat_outside_fib_t *outside_fib;
3387       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3388       fib_prefix_t pfx = {
3389         .fp_proto = FIB_PROTOCOL_IP4,
3390         .fp_len = 32,
3391         .fp_addr = {
3392                     .ip4.as_u32 = ip->dst_address.as_u32,
3393                     }
3394         ,
3395       };
3396
3397       udp = ip4_next_header (ip);
3398
3399       switch (vec_len (sm->outside_fibs))
3400         {
3401         case 0:
3402           fib_index = sm->outside_fib_index;
3403           break;
3404         case 1:
3405           fib_index = sm->outside_fibs[0].fib_index;
3406           break;
3407         default:
3408             /* *INDENT-OFF* */
3409             vec_foreach (outside_fib, sm->outside_fibs)
3410               {
3411                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3412                 if (FIB_NODE_INDEX_INVALID != fei)
3413                   {
3414                     if (fib_entry_get_resolving_interface (fei) != ~0)
3415                       {
3416                         fib_index = outside_fib->fib_index;
3417                         break;
3418                       }
3419                   }
3420               }
3421             /* *INDENT-ON* */
3422           break;
3423         }
3424
3425       make_ed_kv (&ip->src_address, &ip->dst_address,
3426                   ip->protocol, fib_index, udp->src_port, udp->dst_port,
3427                   ~0ULL, &kv16);
3428
3429       /* *INDENT-OFF* */
3430       vec_foreach (tsm, sm->per_thread_data)
3431         {
3432           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3433                                                       &kv16, &value16)))
3434             {
3435               next_worker_index += tsm->thread_index;
3436
3437               nat_elog_debug_handoff (
3438                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3439                 next_worker_index, fib_index,
3440                 clib_net_to_host_u32 (ip->src_address.as_u32),
3441                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3442
3443               return next_worker_index;
3444             }
3445         }
3446       /* *INDENT-ON* */
3447     }
3448
3449   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3450     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3451
3452   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3453     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3454   else
3455     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3456
3457   if (PREDICT_TRUE (!is_output))
3458     {
3459       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3460                               next_worker_index, rx_fib_index,
3461                               clib_net_to_host_u32 (ip->src_address.as_u32),
3462                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3463     }
3464   else
3465     {
3466       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3467                               next_worker_index, rx_fib_index,
3468                               clib_net_to_host_u32 (ip->src_address.as_u32),
3469                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3470     }
3471
3472   return next_worker_index;
3473 }
3474
3475 static u32
3476 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3477                                u32 rx_fib_index, u8 is_output)
3478 {
3479   snat_main_t *sm = &snat_main;
3480   clib_bihash_kv_8_8_t kv, value;
3481   clib_bihash_kv_16_8_t kv16, value16;
3482   snat_main_per_thread_data_t *tsm;
3483
3484   u32 proto, next_worker_index = 0;
3485   udp_header_t *udp;
3486   u16 port;
3487   snat_static_mapping_t *m;
3488   u32 hash;
3489
3490   proto = ip_proto_to_snat_proto (ip->protocol);
3491
3492   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3493     {
3494       udp = ip4_next_header (ip);
3495
3496       make_ed_kv (&ip->dst_address, &ip->src_address,
3497                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port,
3498                   ~0ULL, &kv16);
3499
3500       /* *INDENT-OFF* */
3501       vec_foreach (tsm, sm->per_thread_data)
3502         {
3503           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3504                                                       &kv16, &value16)))
3505             {
3506               next_worker_index = sm->first_worker_index + tsm->thread_index;
3507               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3508                           next_worker_index, rx_fib_index,
3509                           clib_net_to_host_u32 (ip->src_address.as_u32),
3510                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3511               return next_worker_index;
3512             }
3513           }
3514         /* *INDENT-ON* */
3515     }
3516   else if (proto == SNAT_PROTOCOL_ICMP)
3517     {
3518       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0ULL, 0, 0, 0, &kv16))
3519         {
3520           /* *INDENT-OFF* */
3521           vec_foreach (tsm, sm->per_thread_data)
3522             {
3523               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3524                                                           &kv16, &value16)))
3525                 {
3526                   next_worker_index = sm->first_worker_index +
3527                                       tsm->thread_index;
3528                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3529                               next_worker_index, rx_fib_index,
3530                               clib_net_to_host_u32 (ip->src_address.as_u32),
3531                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3532                   return next_worker_index;
3533                 }
3534             }
3535           /* *INDENT-ON* */
3536         }
3537     }
3538
3539   /* first try static mappings without port */
3540   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3541     {
3542       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3543       if (!clib_bihash_search_8_8
3544           (&sm->static_mapping_by_external, &kv, &value))
3545         {
3546           m = pool_elt_at_index (sm->static_mappings, value.value);
3547           next_worker_index = m->workers[0];
3548           goto done;
3549         }
3550     }
3551
3552   /* unknown protocol */
3553   if (PREDICT_FALSE (proto == ~0))
3554     {
3555       /* use current thread */
3556       next_worker_index = vlib_get_thread_index ();
3557       goto done;
3558     }
3559
3560   udp = ip4_next_header (ip);
3561   port = udp->dst_port;
3562
3563   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3564     {
3565       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3566       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3567       if (!icmp_type_is_error_message
3568           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3569         port = vnet_buffer (b)->ip.reass.l4_src_port;
3570       else
3571         {
3572           /* if error message, then it's not fragmented and we can access it */
3573           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3574           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3575           void *l4_header = ip4_next_header (inner_ip);
3576           switch (proto)
3577             {
3578             case SNAT_PROTOCOL_ICMP:
3579               icmp = (icmp46_header_t *) l4_header;
3580               echo = (icmp_echo_header_t *) (icmp + 1);
3581               port = echo->identifier;
3582               break;
3583             case SNAT_PROTOCOL_UDP:
3584             case SNAT_PROTOCOL_TCP:
3585               port = ((tcp_udp_header_t *) l4_header)->src_port;
3586               break;
3587             default:
3588               next_worker_index = vlib_get_thread_index ();
3589               goto done;
3590             }
3591         }
3592     }
3593
3594   /* try static mappings with port */
3595   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3596     {
3597       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3598                   clib_net_to_host_u16 (port));
3599       if (!clib_bihash_search_8_8
3600           (&sm->static_mapping_by_external, &kv, &value))
3601         {
3602           m = pool_elt_at_index (sm->static_mappings, value.value);
3603           if (!is_lb_static_mapping (m))
3604             {
3605               next_worker_index = m->workers[0];
3606               goto done;
3607             }
3608
3609           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3610             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3611
3612           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3613             next_worker_index =
3614               m->workers[hash & (_vec_len (m->workers) - 1)];
3615           else
3616             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3617           goto done;
3618         }
3619     }
3620
3621   /* worker by outside port */
3622   next_worker_index = sm->first_worker_index;
3623   next_worker_index +=
3624     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3625
3626 done:
3627   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3628                           clib_net_to_host_u32 (ip->src_address.as_u32),
3629                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3630   return next_worker_index;
3631 }
3632
3633 void
3634 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3635                 ip4_address_t * out_addr, u16 out_port,
3636                 ip4_address_t * eh_addr, u16 eh_port,
3637                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3638                 u32 fib_index, u16 flags, u32 thread_index)
3639 {
3640   snat_main_t *sm = &snat_main;
3641   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3642   snat_session_key_t key;
3643   snat_user_t *u;
3644   snat_session_t *s;
3645   clib_bihash_kv_8_8_t kv;
3646   f64 now = vlib_time_now (tsm->vlib_main);
3647   nat_outside_fib_t *outside_fib;
3648   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3649   fib_prefix_t pfx = {
3650     .fp_proto = FIB_PROTOCOL_IP4,
3651     .fp_len = 32,
3652     .fp_addr = {
3653                 .ip4.as_u32 = eh_addr->as_u32,
3654                 },
3655   };
3656
3657   key.addr.as_u32 = out_addr->as_u32;
3658   key.port = out_port;
3659   key.protocol = proto;
3660
3661   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3662     {
3663       if (nat_set_outside_address_and_port
3664           (sm->addresses, thread_index, &key))
3665         return;
3666     }
3667
3668   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3669   if (!u)
3670     return;
3671
3672   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3673   if (!s)
3674     return;
3675
3676   s->last_heard = now;
3677   s->flags = flags;
3678   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3679   s->ext_host_port = eh_port;
3680   user_session_increment (sm, u, snat_is_session_static (s));
3681   switch (vec_len (sm->outside_fibs))
3682     {
3683     case 0:
3684       key.fib_index = sm->outside_fib_index;
3685       break;
3686     case 1:
3687       key.fib_index = sm->outside_fibs[0].fib_index;
3688       break;
3689     default:
3690       /* *INDENT-OFF* */
3691       vec_foreach (outside_fib, sm->outside_fibs)
3692         {
3693           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3694           if (FIB_NODE_INDEX_INVALID != fei)
3695             {
3696               if (fib_entry_get_resolving_interface (fei) != ~0)
3697                 {
3698                   key.fib_index = outside_fib->fib_index;
3699                   break;
3700                 }
3701             }
3702         }
3703       /* *INDENT-ON* */
3704       break;
3705     }
3706   s->out2in = key;
3707   kv.key = key.as_u64;
3708   kv.value = s - tsm->sessions;
3709   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3710     nat_elog_warn ("out2in key add failed");
3711
3712   key.addr.as_u32 = in_addr->as_u32;
3713   key.port = in_port;
3714   key.fib_index = fib_index;
3715   s->in2out = key;
3716   kv.key = key.as_u64;
3717   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3718     nat_elog_warn ("in2out key add failed");
3719 }
3720
3721 void
3722 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3723                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3724                 u32 ti)
3725 {
3726   snat_main_t *sm = &snat_main;
3727   snat_session_key_t key;
3728   clib_bihash_kv_8_8_t kv, value;
3729   u32 thread_index;
3730   snat_session_t *s;
3731   snat_main_per_thread_data_t *tsm;
3732
3733   if (sm->num_workers > 1)
3734     thread_index =
3735       sm->first_worker_index +
3736       (sm->workers[(clib_net_to_host_u16 (out_port) -
3737                     1024) / sm->port_per_thread]);
3738   else
3739     thread_index = sm->num_workers;
3740   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3741
3742   key.addr.as_u32 = out_addr->as_u32;
3743   key.port = out_port;
3744   key.protocol = proto;
3745   key.fib_index = fib_index;
3746   kv.key = key.as_u64;
3747   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3748     return;
3749
3750   s = pool_elt_at_index (tsm->sessions, value.value);
3751   nat_free_session_data (sm, s, thread_index, 1);
3752   nat44_delete_session (sm, s, thread_index);
3753 }
3754
3755 void
3756 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3757                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3758                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3759 {
3760   snat_main_t *sm = &snat_main;
3761   snat_session_key_t key;
3762   clib_bihash_kv_8_8_t kv, value;
3763   snat_session_t *s;
3764   snat_main_per_thread_data_t *tsm;
3765
3766   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3767
3768   key.addr.as_u32 = out_addr->as_u32;
3769   key.port = out_port;
3770   key.protocol = proto;
3771   key.fib_index = fib_index;
3772   kv.key = key.as_u64;
3773   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3774     return;
3775
3776   s = pool_elt_at_index (tsm->sessions, value.value);
3777   s->total_pkts = total_pkts;
3778   s->total_bytes = total_bytes;
3779 }
3780
3781 void
3782 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3783                    ip4_address_t * out_addr, u16 out_port,
3784                    ip4_address_t * eh_addr, u16 eh_port,
3785                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3786                    u32 fib_index, u16 flags, u32 thread_index)
3787 {
3788   snat_main_t *sm = &snat_main;
3789   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3790   snat_session_key_t key;
3791   snat_session_t *s;
3792   clib_bihash_kv_16_8_t kv;
3793   f64 now = vlib_time_now (tsm->vlib_main);
3794   nat_outside_fib_t *outside_fib;
3795   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3796   fib_prefix_t pfx = {
3797     .fp_proto = FIB_PROTOCOL_IP4,
3798     .fp_len = 32,
3799     .fp_addr = {
3800                 .ip4.as_u32 = eh_addr->as_u32,
3801                 },
3802   };
3803
3804   key.addr.as_u32 = out_addr->as_u32;
3805   key.port = out_port;
3806   key.protocol = proto;
3807
3808   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3809     {
3810       if (nat_set_outside_address_and_port
3811           (sm->addresses, thread_index, &key))
3812         return;
3813     }
3814
3815   key.addr.as_u32 = ehn_addr->as_u32;
3816   key.port = ehn_port;
3817   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3818     {
3819       if (nat_set_outside_address_and_port
3820           (sm->twice_nat_addresses, thread_index, &key))
3821         return;
3822     }
3823
3824   s = nat_ed_session_alloc (sm, thread_index, now);
3825   if (!s)
3826     return;
3827
3828   s->last_heard = now;
3829   s->flags = flags;
3830   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3831   s->ext_host_nat_port = s->ext_host_port = eh_port;
3832   if (is_twice_nat_session (s))
3833     {
3834       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3835       s->ext_host_nat_port = ehn_port;
3836     }
3837   switch (vec_len (sm->outside_fibs))
3838     {
3839     case 0:
3840       key.fib_index = sm->outside_fib_index;
3841       break;
3842     case 1:
3843       key.fib_index = sm->outside_fibs[0].fib_index;
3844       break;
3845     default:
3846       /* *INDENT-OFF* */
3847       vec_foreach (outside_fib, sm->outside_fibs)
3848         {
3849           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3850           if (FIB_NODE_INDEX_INVALID != fei)
3851             {
3852               if (fib_entry_get_resolving_interface (fei) != ~0)
3853                 {
3854                   key.fib_index = outside_fib->fib_index;
3855                   break;
3856                 }
3857             }
3858         }
3859       /* *INDENT-ON* */
3860       break;
3861     }
3862   key.addr.as_u32 = out_addr->as_u32;
3863   key.port = out_port;
3864   s->out2in = key;
3865   kv.value = s - tsm->sessions;
3866
3867   key.addr.as_u32 = in_addr->as_u32;
3868   key.port = in_port;
3869   key.fib_index = fib_index;
3870   s->in2out = key;
3871
3872   make_ed_kv (in_addr, &s->ext_host_nat_addr,
3873               snat_proto_to_ip_proto (proto), fib_index, in_port,
3874               s->ext_host_nat_port, s - tsm->sessions, &kv);
3875   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3876     nat_elog_warn ("in2out key add failed");
3877
3878   make_ed_kv (out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3879               s->out2in.fib_index, out_port, eh_port, s - tsm->sessions, &kv);
3880   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3881     nat_elog_warn ("out2in key add failed");
3882 }
3883
3884 void
3885 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3886                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3887                    u32 fib_index, u32 ti)
3888 {
3889   snat_main_t *sm = &snat_main;
3890   clib_bihash_kv_16_8_t kv, value;
3891   u32 thread_index;
3892   snat_session_t *s;
3893   snat_main_per_thread_data_t *tsm;
3894
3895   if (sm->num_workers > 1)
3896     thread_index =
3897       sm->first_worker_index +
3898       (sm->workers[(clib_net_to_host_u16 (out_port) -
3899                     1024) / sm->port_per_thread]);
3900   else
3901     thread_index = sm->num_workers;
3902   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3903
3904   make_ed_kv (out_addr, eh_addr, proto, fib_index, out_port, eh_port, ~0ULL,
3905               &kv);
3906   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3907     return;
3908
3909   s = pool_elt_at_index (tsm->sessions, value.value);
3910   nat_free_session_data (sm, s, thread_index, 1);
3911   nat44_delete_session (sm, s, thread_index);
3912 }
3913
3914 void
3915 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3916                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3917                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3918                    u32 thread_index)
3919 {
3920   snat_main_t *sm = &snat_main;
3921   clib_bihash_kv_16_8_t kv, value;
3922   snat_session_t *s;
3923   snat_main_per_thread_data_t *tsm;
3924
3925   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3926
3927   make_ed_kv (out_addr, eh_addr, proto, fib_index, out_port, eh_port, ~0ULL,
3928               &kv);
3929   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3930     return;
3931
3932   s = pool_elt_at_index (tsm->sessions, value.value);
3933   s->total_pkts = total_pkts;
3934   s->total_bytes = total_bytes;
3935 }
3936
3937 void
3938 nat44_db_init (snat_main_per_thread_data_t * tsm)
3939 {
3940   snat_main_t *sm = &snat_main;
3941
3942   pool_alloc (tsm->sessions, sm->max_translations);
3943   pool_alloc (tsm->global_lru_pool, sm->max_translations);
3944
3945   dlist_elt_t *head;
3946   pool_get (tsm->global_lru_pool, head);
3947   tsm->global_lru_head_index = head - tsm->global_lru_pool;
3948   clib_dlist_init (tsm->global_lru_pool, tsm->global_lru_head_index);
3949
3950   if (sm->endpoint_dependent)
3951     {
3952       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3953                              sm->translation_buckets,
3954                              sm->translation_memory_size);
3955       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3956                                           format_ed_session_kvp);
3957       clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3958                              sm->translation_buckets,
3959                              sm->translation_memory_size);
3960       clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3961                                           format_ed_session_kvp);
3962     }
3963   else
3964     {
3965       clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3966                             sm->translation_buckets,
3967                             sm->translation_memory_size);
3968       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
3969       clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3970                             sm->translation_buckets,
3971                             sm->translation_memory_size);
3972       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
3973     }
3974
3975   // TODO: resolve static mappings (put only to !ED)
3976   pool_alloc (tsm->list_pool, sm->max_translations);
3977   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets,
3978                         sm->user_memory_size);
3979   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3980 }
3981
3982 void
3983 nat44_db_free (snat_main_per_thread_data_t * tsm)
3984 {
3985   snat_main_t *sm = &snat_main;
3986
3987   pool_free (tsm->sessions);
3988   pool_free (tsm->global_lru_pool);
3989
3990   if (sm->endpoint_dependent)
3991     {
3992       clib_bihash_free_16_8 (&tsm->in2out_ed);
3993       clib_bihash_free_16_8 (&tsm->out2in_ed);
3994     }
3995   else
3996     {
3997       clib_bihash_free_8_8 (&tsm->in2out);
3998       clib_bihash_free_8_8 (&tsm->out2in);
3999     }
4000
4001   // TODO: resolve static mappings (put only to !ED)
4002   pool_free (tsm->users);
4003   pool_free (tsm->list_pool);
4004   clib_bihash_free_8_8 (&tsm->user_hash);
4005 }
4006
4007 static clib_error_t *
4008 snat_config (vlib_main_t * vm, unformat_input_t * input)
4009 {
4010   snat_main_t *sm = &snat_main;
4011   nat66_main_t *nm = &nat66_main;
4012   //dslite_main_t *dm = &dslite_main;
4013   snat_main_per_thread_data_t *tsm;
4014
4015   u32 static_mapping_buckets = 1024;
4016   uword static_mapping_memory_size = 64 << 20;
4017
4018   u32 nat64_bib_buckets = 1024;
4019   u32 nat64_bib_memory_size = 128 << 20;
4020
4021   u32 nat64_st_buckets = 2048;
4022   uword nat64_st_memory_size = 256 << 20;
4023
4024   u32 user_buckets = 128;
4025   uword user_memory_size = 64 << 20;
4026   u32 translation_buckets = 1024;
4027   uword translation_memory_size = 128 << 20;
4028
4029   u32 max_translations_per_user = ~0;
4030
4031   u32 outside_vrf_id = 0;
4032   u32 outside_ip6_vrf_id = 0;
4033   u32 inside_vrf_id = 0;
4034   u8 static_mapping_only = 0;
4035   u8 static_mapping_connection_tracking = 0;
4036
4037   // configurable timeouts
4038   u32 udp_timeout = SNAT_UDP_TIMEOUT;
4039   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
4040   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
4041   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
4042
4043   sm->deterministic = 0;
4044   sm->out2in_dpo = 0;
4045   sm->endpoint_dependent = 0;
4046
4047   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
4048     {
4049       if (unformat
4050           (input, "translation hash buckets %d", &translation_buckets))
4051         ;
4052       else if (unformat (input, "udp timeout %d", &udp_timeout))
4053         ;
4054       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
4055         ;
4056       else if (unformat (input, "tcp transitory timeout %d",
4057                          &tcp_transitory_timeout));
4058       else if (unformat (input, "tcp established timeout %d",
4059                          &tcp_established_timeout));
4060       else if (unformat (input, "translation hash memory %d",
4061                          &translation_memory_size));
4062       else if (unformat (input, "user hash buckets %d", &user_buckets))
4063         ;
4064       else if (unformat (input, "user hash memory %d", &user_memory_size))
4065         ;
4066       else if (unformat (input, "max translations per user %d",
4067                          &max_translations_per_user))
4068         ;
4069       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
4070         ;
4071       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
4072         ;
4073       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
4074         ;
4075       else if (unformat (input, "static mapping only"))
4076         {
4077           static_mapping_only = 1;
4078           if (unformat (input, "connection tracking"))
4079             static_mapping_connection_tracking = 1;
4080         }
4081       else if (unformat (input, "deterministic"))
4082         sm->deterministic = 1;
4083       else if (unformat (input, "nat64 bib hash buckets %d",
4084                          &nat64_bib_buckets))
4085         ;
4086       else if (unformat (input, "nat64 bib hash memory %d",
4087                          &nat64_bib_memory_size))
4088         ;
4089       else
4090         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
4091         ;
4092       else if (unformat (input, "nat64 st hash memory %d",
4093                          &nat64_st_memory_size))
4094         ;
4095       else if (unformat (input, "out2in dpo"))
4096         sm->out2in_dpo = 1;
4097       //else if (unformat (input, "dslite ce"))
4098       //dslite_set_ce (dm, 1);
4099       else if (unformat (input, "endpoint-dependent"))
4100         sm->endpoint_dependent = 1;
4101       else
4102         return clib_error_return (0, "unknown input '%U'",
4103                                   format_unformat_error, input);
4104     }
4105
4106   if (sm->deterministic && sm->endpoint_dependent)
4107     return clib_error_return (0,
4108                               "deterministic and endpoint-dependent modes are mutually exclusive");
4109
4110   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
4111     return clib_error_return (0,
4112                               "static mapping only mode available only for simple nat");
4113
4114   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
4115     return clib_error_return (0,
4116                               "out2in dpo mode available only for simple nat");
4117
4118   /* optionally configurable timeouts for testing purposes */
4119   sm->udp_timeout = udp_timeout;
4120   sm->tcp_transitory_timeout = tcp_transitory_timeout;
4121   sm->tcp_established_timeout = tcp_established_timeout;
4122   sm->icmp_timeout = icmp_timeout;
4123
4124   sm->user_buckets = user_buckets;
4125   sm->user_memory_size = user_memory_size;
4126
4127   sm->translation_buckets = translation_buckets;
4128   sm->translation_memory_size = translation_memory_size;
4129   /* do not exceed load factor 10 */
4130   sm->max_translations = 10 * translation_buckets;
4131   vec_add1 (sm->max_translations_per_fib, sm->max_translations);
4132
4133   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4134     sm->max_translations : max_translations_per_user;
4135
4136   sm->outside_vrf_id = outside_vrf_id;
4137   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4138                                                              outside_vrf_id,
4139                                                              nat_fib_src_hi);
4140   nm->outside_vrf_id = outside_ip6_vrf_id;
4141   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
4142                                                              outside_ip6_vrf_id,
4143                                                              nat_fib_src_hi);
4144   sm->inside_vrf_id = inside_vrf_id;
4145   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4146                                                             inside_vrf_id,
4147                                                             nat_fib_src_hi);
4148   sm->static_mapping_only = static_mapping_only;
4149   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4150
4151   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4152                   nat64_st_memory_size);
4153
4154   if (sm->deterministic)
4155     {
4156       sm->in2out_node_index = snat_det_in2out_node.index;
4157       sm->in2out_output_node_index = ~0;
4158       sm->out2in_node_index = snat_det_out2in_node.index;
4159       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4160       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4161     }
4162   else
4163     {
4164       if (sm->endpoint_dependent)
4165         {
4166           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4167           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4168
4169           sm->handoff_out2in_index = nat_pre_out2in_node.index;
4170           sm->handoff_in2out_index = nat_pre_in2out_node.index;
4171           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
4172
4173           sm->in2out_node_index = nat44_ed_in2out_node.index;
4174           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4175           sm->out2in_node_index = nat44_ed_out2in_node.index;
4176
4177           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4178           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4179           nat_affinity_init (vm);
4180           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4181                        nat_ha_sref_ed_cb);
4182         }
4183       else
4184         {
4185           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4186           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4187
4188           sm->handoff_out2in_index = snat_out2in_node.index;
4189           sm->handoff_in2out_index = snat_in2out_node.index;
4190           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
4191
4192           sm->in2out_node_index = snat_in2out_node.index;
4193           sm->in2out_output_node_index = snat_in2out_output_node.index;
4194           sm->out2in_node_index = snat_out2in_node.index;
4195           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4196           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4197           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4198         }
4199       if (!static_mapping_only ||
4200           (static_mapping_only && static_mapping_connection_tracking))
4201         {
4202           /* *INDENT-OFF* */
4203           vec_foreach (tsm, sm->per_thread_data)
4204             {
4205               nat44_db_init (tsm);
4206             }
4207           /* *INDENT-ON* */
4208         }
4209       else
4210         {
4211           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4212           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4213         }
4214       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4215                             "static_mapping_by_local", static_mapping_buckets,
4216                             static_mapping_memory_size);
4217       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4218                                          format_static_mapping_kvp);
4219
4220       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4221                             "static_mapping_by_external",
4222                             static_mapping_buckets,
4223                             static_mapping_memory_size);
4224       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4225                                          format_static_mapping_kvp);
4226     }
4227
4228   return 0;
4229 }
4230
4231 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4232
4233 static void
4234 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4235                                  uword opaque,
4236                                  u32 sw_if_index,
4237                                  ip4_address_t * address,
4238                                  u32 address_length,
4239                                  u32 if_address_index, u32 is_delete)
4240 {
4241   snat_main_t *sm = &snat_main;
4242   snat_static_map_resolve_t *rp;
4243   snat_static_mapping_t *m;
4244   snat_session_key_t m_key;
4245   clib_bihash_kv_8_8_t kv, value;
4246   int i, rv;
4247   ip4_address_t l_addr;
4248
4249   for (i = 0; i < vec_len (sm->to_resolve); i++)
4250     {
4251       rp = sm->to_resolve + i;
4252       if (rp->addr_only == 0)
4253         continue;
4254       if (rp->sw_if_index == sw_if_index)
4255         goto match;
4256     }
4257
4258   return;
4259
4260 match:
4261   m_key.addr.as_u32 = address->as_u32;
4262   m_key.port = rp->addr_only ? 0 : rp->e_port;
4263   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4264   m_key.fib_index = sm->outside_fib_index;
4265   kv.key = m_key.as_u64;
4266   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4267     m = 0;
4268   else
4269     m = pool_elt_at_index (sm->static_mappings, value.value);
4270
4271   if (!is_delete)
4272     {
4273       /* Don't trip over lease renewal, static config */
4274       if (m)
4275         return;
4276     }
4277   else
4278     {
4279       if (!m)
4280         return;
4281     }
4282
4283   /* Indetity mapping? */
4284   if (rp->l_addr.as_u32 == 0)
4285     l_addr.as_u32 = address[0].as_u32;
4286   else
4287     l_addr.as_u32 = rp->l_addr.as_u32;
4288   /* Add the static mapping */
4289   rv = snat_add_static_mapping (l_addr,
4290                                 address[0],
4291                                 rp->l_port,
4292                                 rp->e_port,
4293                                 rp->vrf_id,
4294                                 rp->addr_only, ~0 /* sw_if_index */ ,
4295                                 rp->proto, !is_delete, rp->twice_nat,
4296                                 rp->out2in_only, rp->tag, rp->identity_nat);
4297   if (rv)
4298     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4299 }
4300
4301 static void
4302 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4303                                        uword opaque,
4304                                        u32 sw_if_index,
4305                                        ip4_address_t * address,
4306                                        u32 address_length,
4307                                        u32 if_address_index, u32 is_delete)
4308 {
4309   snat_main_t *sm = &snat_main;
4310   snat_static_map_resolve_t *rp;
4311   ip4_address_t l_addr;
4312   int i, j;
4313   int rv;
4314   u8 twice_nat = 0;
4315   snat_address_t *addresses = sm->addresses;
4316
4317   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4318     {
4319       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4320         goto match;
4321     }
4322
4323   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4324     {
4325       twice_nat = 1;
4326       addresses = sm->twice_nat_addresses;
4327       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4328         goto match;
4329     }
4330
4331   return;
4332
4333 match:
4334   if (!is_delete)
4335     {
4336       /* Don't trip over lease renewal, static config */
4337       for (j = 0; j < vec_len (addresses); j++)
4338         if (addresses[j].addr.as_u32 == address->as_u32)
4339           return;
4340
4341       (void) snat_add_address (sm, address, ~0, twice_nat);
4342       /* Scan static map resolution vector */
4343       for (j = 0; j < vec_len (sm->to_resolve); j++)
4344         {
4345           rp = sm->to_resolve + j;
4346           if (rp->addr_only)
4347             continue;
4348           /* On this interface? */
4349           if (rp->sw_if_index == sw_if_index)
4350             {
4351               /* Indetity mapping? */
4352               if (rp->l_addr.as_u32 == 0)
4353                 l_addr.as_u32 = address[0].as_u32;
4354               else
4355                 l_addr.as_u32 = rp->l_addr.as_u32;
4356               /* Add the static mapping */
4357               rv = snat_add_static_mapping (l_addr,
4358                                             address[0],
4359                                             rp->l_port,
4360                                             rp->e_port,
4361                                             rp->vrf_id,
4362                                             rp->addr_only,
4363                                             ~0 /* sw_if_index */ ,
4364                                             rp->proto,
4365                                             rp->is_add, rp->twice_nat,
4366                                             rp->out2in_only, rp->tag,
4367                                             rp->identity_nat);
4368               if (rv)
4369                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4370                                     "i4", rv);
4371             }
4372         }
4373       return;
4374     }
4375   else
4376     {
4377       (void) snat_del_address (sm, address[0], 1, twice_nat);
4378       return;
4379     }
4380 }
4381
4382
4383 int
4384 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4385                             u8 twice_nat)
4386 {
4387   ip4_main_t *ip4_main = sm->ip4_main;
4388   ip4_address_t *first_int_addr;
4389   snat_static_map_resolve_t *rp;
4390   u32 *indices_to_delete = 0;
4391   int i, j;
4392   u32 *auto_add_sw_if_indices =
4393     twice_nat ? sm->
4394     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4395
4396   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4397     );
4398
4399   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4400     {
4401       if (auto_add_sw_if_indices[i] == sw_if_index)
4402         {
4403           if (is_del)
4404             {
4405               /* if have address remove it */
4406               if (first_int_addr)
4407                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4408               else
4409                 {
4410                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4411                     {
4412                       rp = sm->to_resolve + j;
4413                       if (rp->sw_if_index == sw_if_index)
4414                         vec_add1 (indices_to_delete, j);
4415                     }
4416                   if (vec_len (indices_to_delete))
4417                     {
4418                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4419                         vec_del1 (sm->to_resolve, j);
4420                       vec_free (indices_to_delete);
4421                     }
4422                 }
4423               if (twice_nat)
4424                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4425               else
4426                 vec_del1 (sm->auto_add_sw_if_indices, i);
4427             }
4428           else
4429             return VNET_API_ERROR_VALUE_EXIST;
4430
4431           return 0;
4432         }
4433     }
4434
4435   if (is_del)
4436     return VNET_API_ERROR_NO_SUCH_ENTRY;
4437
4438   /* add to the auto-address list */
4439   if (twice_nat)
4440     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4441   else
4442     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4443
4444   /* If the address is already bound - or static - add it now */
4445   if (first_int_addr)
4446     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4447
4448   return 0;
4449 }
4450
4451 int
4452 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4453                    snat_protocol_t proto, u32 vrf_id, int is_in)
4454 {
4455   snat_main_per_thread_data_t *tsm;
4456   clib_bihash_kv_8_8_t kv, value;
4457   ip4_header_t ip;
4458   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4459   snat_session_key_t key;
4460   snat_session_t *s;
4461   clib_bihash_8_8_t *t;
4462
4463   if (sm->endpoint_dependent)
4464     return VNET_API_ERROR_UNSUPPORTED;
4465
4466   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4467   if (sm->num_workers > 1)
4468     tsm =
4469       vec_elt_at_index (sm->per_thread_data,
4470                         sm->worker_in2out_cb (&ip, fib_index, 0));
4471   else
4472     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4473
4474   key.addr.as_u32 = addr->as_u32;
4475   key.port = clib_host_to_net_u16 (port);
4476   key.protocol = proto;
4477   key.fib_index = fib_index;
4478   kv.key = key.as_u64;
4479   t = is_in ? &tsm->in2out : &tsm->out2in;
4480   if (!clib_bihash_search_8_8 (t, &kv, &value))
4481     {
4482       if (pool_is_free_index (tsm->sessions, value.value))
4483         return VNET_API_ERROR_UNSPECIFIED;
4484
4485       s = pool_elt_at_index (tsm->sessions, value.value);
4486       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4487       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4488       return 0;
4489     }
4490
4491   return VNET_API_ERROR_NO_SUCH_ENTRY;
4492 }
4493
4494 int
4495 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4496                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4497                       u32 vrf_id, int is_in)
4498 {
4499   ip4_header_t ip;
4500   clib_bihash_16_8_t *t;
4501   clib_bihash_kv_16_8_t kv, value;
4502   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4503   snat_session_t *s;
4504   snat_main_per_thread_data_t *tsm;
4505
4506   if (!sm->endpoint_dependent)
4507     return VNET_API_ERROR_FEATURE_DISABLED;
4508
4509   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4510   if (sm->num_workers > 1)
4511     tsm =
4512       vec_elt_at_index (sm->per_thread_data,
4513                         sm->worker_in2out_cb (&ip, fib_index, 0));
4514   else
4515     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4516
4517   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4518   make_ed_kv (addr, eh_addr, proto, fib_index, clib_host_to_net_u16 (port),
4519               clib_host_to_net_u16 (eh_port), ~0ULL, &kv);
4520   if (clib_bihash_search_16_8 (t, &kv, &value))
4521     {
4522       return VNET_API_ERROR_NO_SUCH_ENTRY;
4523     }
4524
4525   if (pool_is_free_index (tsm->sessions, value.value))
4526     return VNET_API_ERROR_UNSPECIFIED;
4527   s = pool_elt_at_index (tsm->sessions, value.value);
4528   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4529   nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
4530   return 0;
4531 }
4532
4533 void
4534 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4535 {
4536   snat_main_t *sm = &snat_main;
4537
4538   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4539   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4540   sm->psid = psid;
4541   sm->psid_offset = psid_offset;
4542   sm->psid_length = psid_length;
4543 }
4544
4545 void
4546 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4547 {
4548   snat_main_t *sm = &snat_main;
4549
4550   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4551   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4552   sm->start_port = start_port;
4553   sm->end_port = end_port;
4554 }
4555
4556 void
4557 nat_set_alloc_addr_and_port_default (void)
4558 {
4559   snat_main_t *sm = &snat_main;
4560
4561   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4562   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4563 }
4564
4565 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4566                                  vlib_node_runtime_t * node,
4567                                  vlib_frame_t * frame)
4568 {
4569   return 0;
4570 }
4571
4572 /* *INDENT-OFF* */
4573 VLIB_REGISTER_NODE (nat_default_node) = {
4574   .name = "nat-default",
4575   .vector_size = sizeof (u32),
4576   .format_trace = 0,
4577   .type = VLIB_NODE_TYPE_INTERNAL,
4578   .n_errors = 0,
4579   .n_next_nodes = NAT_N_NEXT,
4580   .next_nodes = {
4581     [NAT_NEXT_DROP] = "error-drop",
4582     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4583     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4584     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4585     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4586     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4587     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4588     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4589     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4590   },
4591 };
4592 /* *INDENT-ON* */
4593
4594 /*
4595  * fd.io coding-style-patch-verification: ON
4596  *
4597  * Local Variables:
4598  * eval: (c-set-style "gnu")
4599  * End:
4600  */