src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36 #include <vppinfra/bihash_16_8.h>
  37 #include <nat/nat44/ed_inlines.h>
  38
  39 #include <vpp/app/version.h>
  40
  41 snat_main_t snat_main;
  42
  43 fib_source_t nat_fib_src_hi;
  44 fib_source_t nat_fib_src_low;
  45
  46 /* *INDENT-OFF* */
  47 /* Hook up input features */
  48 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat-pre-in2out",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-sv-reassembly-feature"),
  53 };
  54 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat-pre-out2in",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  58                                "ip4-dhcp-client-detect",
  59                                "ip4-sv-reassembly-feature"),
  60 };
  61 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  62   .arc_name = "ip4-unicast",
  63   .node_name = "nat44-in2out-worker-handoff",
  64   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  65 };
  66 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  67   .arc_name = "ip4-unicast",
  68   .node_name = "nat44-out2in-worker-handoff",
  69   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  70                                "ip4-dhcp-client-detect"),
  71 };
  72 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  73   .arc_name = "ip4-unicast",
  74   .node_name = "nat44-in2out",
  75   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  76 };
  77 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  78   .arc_name = "ip4-unicast",
  79   .node_name = "nat44-out2in",
  80   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  81                                "ip4-dhcp-client-detect"),
  82 };
  83 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-det-in2out",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  92 };
  93 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  94   .arc_name = "ip4-unicast",
  95   .node_name = "nat44-det-out2in",
  96   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  97                                "ip4-dhcp-client-detect"),
  98 };
  99 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
 100   .arc_name = "ip4-unicast",
 101   .node_name = "nat44-det-classify",
 102   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-ed-in2out",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 108 };
 109 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-ed-out2in",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-classify",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-handoff-classify",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 124 };
 125 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 126   .arc_name = "ip4-unicast",
 127   .node_name = "nat44-in2out-fast",
 128   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 129 };
 130 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 131   .arc_name = "ip4-unicast",
 132   .node_name = "nat44-out2in-fast",
 133   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 134                                "ip4-dhcp-client-detect"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 137   .arc_name = "ip4-unicast",
 138   .node_name = "nat44-hairpin-dst",
 139   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 140 };
 141 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 142   .arc_name = "ip4-unicast",
 143   .node_name = "nat44-ed-hairpin-dst",
 144   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 145 };
 146
 147 /* Hook up output features */
 148 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-in2out-output",
 151   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 152 };
 153 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 154   .arc_name = "ip4-output",
 155   .node_name = "nat44-in2out-output-worker-handoff",
 156   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 157 };
 158 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 159   .arc_name = "ip4-output",
 160   .node_name = "nat44-hairpin-src",
 161   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 164   .arc_name = "ip4-output",
 165   .node_name = "nat44-ed-in2out-output",
 166   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 167   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 168 };
 169 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 170   .arc_name = "ip4-output",
 171   .node_name = "nat44-ed-hairpin-src",
 172   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 173   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 174 };
 175
 176 /* Hook up ip4-local features */
 177 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 178 {
 179   .arc_name = "ip4-local",
 180   .node_name = "nat44-hairpinning",
 181   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 182 };
 183 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 184 {
 185   .arc_name = "ip4-local",
 186   .node_name = "nat44-ed-hairpinning",
 187   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 188 };
 189
 190
 191 VLIB_PLUGIN_REGISTER () = {
 192     .version = VPP_BUILD_VER,
 193     .description = "Network Address Translation (NAT)",
 194 };
 195 /* *INDENT-ON* */
 196
 197 void
 198 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 199                        u8 is_ha)
 200 {
 201   snat_session_key_t key;
 202   clib_bihash_kv_8_8_t kv;
 203   u8 proto;
 204   u16 r_port, l_port;
 205   ip4_address_t *l_addr, *r_addr;
 206   u32 fib_index = 0;
 207   clib_bihash_kv_16_8_t ed_kv;
 208   snat_main_per_thread_data_t *tsm =
 209     vec_elt_at_index (sm->per_thread_data, thread_index);
 210
 211   if (is_fwd_bypass_session (s))
 212     {
 213       if (snat_is_unk_proto_session (s))
 214         {
 215           make_ed_kv (&s->in2out.addr, &s->ext_host_addr, s->in2out.port, 0,
 216                       0, 0, ~0ULL, &ed_kv);
 217         }
 218       else
 219         {
 220           l_port = s->in2out.port;
 221           r_port = s->ext_host_port;
 222           l_addr = &s->in2out.addr;
 223           r_addr = &s->ext_host_addr;
 224           proto = nat_proto_to_ip_proto (s->in2out.protocol);
 225           make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 226                       &ed_kv);
 227         }
 228       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 229         nat_elog_warn ("in2out_ed key del failed");
 230       return;
 231     }
 232
 233   /* session lookup tables */
 234   if (is_ed_session (s))
 235     {
 236       if (is_affinity_sessions (s))
 237         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 238                              s->in2out.protocol, s->out2in.port);
 239       l_addr = &s->out2in.addr;
 240       r_addr = &s->ext_host_addr;
 241       fib_index = s->out2in.fib_index;
 242       if (snat_is_unk_proto_session (s))
 243         {
 244           proto = s->in2out.port;
 245           r_port = 0;
 246           l_port = 0;
 247         }
 248       else
 249         {
 250           proto = nat_proto_to_ip_proto (s->in2out.protocol);
 251           l_port = s->out2in.port;
 252           r_port = s->ext_host_port;
 253         }
 254       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 255                   &ed_kv);
 256       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 257         nat_elog_warn ("out2in_ed key del failed");
 258       l_addr = &s->in2out.addr;
 259       fib_index = s->in2out.fib_index;
 260       if (!snat_is_unk_proto_session (s))
 261         l_port = s->in2out.port;
 262       if (is_twice_nat_session (s))
 263         {
 264           r_addr = &s->ext_host_nat_addr;
 265           r_port = s->ext_host_nat_port;
 266         }
 267       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 268                   &ed_kv);
 269       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 270         nat_elog_warn ("in2out_ed key del failed");
 271
 272       if (!is_ha)
 273         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 274                                &s->in2out.addr, s->in2out.port,
 275                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 276                                &s->out2in.addr, s->out2in.port,
 277                                &s->ext_host_addr, s->ext_host_port,
 278                                s->in2out.protocol, is_twice_nat_session (s));
 279     }
 280   else
 281     {
 282       kv.key = s->in2out.as_u64;
 283       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 284         nat_elog_warn ("in2out key del failed");
 285       kv.key = s->out2in.as_u64;
 286       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 287         nat_elog_warn ("out2in key del failed");
 288
 289       if (!is_ha)
 290         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 291                                  &s->in2out.addr, s->in2out.port,
 292                                  &s->out2in.addr, s->out2in.port,
 293                                  s->in2out.protocol);
 294     }
 295
 296   if (snat_is_unk_proto_session (s))
 297     return;
 298
 299   if (!is_ha)
 300     {
 301       /* log NAT event */
 302       snat_ipfix_logging_nat44_ses_delete (thread_index,
 303                                            s->in2out.addr.as_u32,
 304                                            s->out2in.addr.as_u32,
 305                                            s->in2out.protocol,
 306                                            s->in2out.port,
 307                                            s->out2in.port,
 308                                            s->in2out.fib_index);
 309
 310       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 311                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 312                    thread_index);
 313     }
 314
 315   /* Twice NAT address and port for external host */
 316   if (is_twice_nat_session (s))
 317     {
 318       key.protocol = s->in2out.protocol;
 319       key.port = s->ext_host_nat_port;
 320       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 321       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 322                                           thread_index, &key);
 323     }
 324
 325   if (snat_is_session_static (s))
 326     return;
 327
 328   snat_free_outside_address_and_port (sm->addresses, thread_index,
 329                                       &s->out2in);
 330 }
 331
 332 int
 333 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
 334 {
 335   snat_main_t *sm = &snat_main;
 336   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
 337   u32 len = vec_len (sm->max_translations_per_fib);
 338
 339   if (len <= fib_index)
 340     {
 341       vec_validate (sm->max_translations_per_fib, fib_index + 1);
 342
 343       for (; len < vec_len (sm->max_translations_per_fib); len++)
 344         sm->max_translations_per_fib[len] = sm->max_translations;
 345     }
 346
 347   sm->max_translations_per_fib[fib_index] = session_limit;
 348   return 0;
 349 }
 350
 351 void
 352 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 353                          u32 thread_index, u8 is_ha)
 354 {
 355   snat_session_key_t key;
 356   u8 proto;
 357   u16 r_port, l_port;
 358   ip4_address_t *l_addr, *r_addr;
 359   u32 fib_index;
 360   clib_bihash_kv_16_8_t ed_kv;
 361   snat_main_per_thread_data_t *tsm =
 362     vec_elt_at_index (sm->per_thread_data, thread_index);
 363
 364   if (is_fwd_bypass_session (s))
 365     {
 366       if (snat_is_unk_proto_session (s))
 367         {
 368           proto = s->in2out.port;
 369           r_port = 0;
 370           l_port = 0;
 371         }
 372       else
 373         {
 374           proto = nat_proto_to_ip_proto (s->in2out.protocol);
 375           l_port = s->in2out.port;
 376           r_port = s->ext_host_port;
 377         }
 378
 379       l_addr = &s->in2out.addr;
 380       r_addr = &s->ext_host_addr;
 381       fib_index = 0;
 382       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 383                   &ed_kv);
 384
 385       if (PREDICT_FALSE
 386           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 387         nat_elog_warn ("in2out_ed key del failed");
 388       return;
 389     }
 390
 391   /* session lookup tables */
 392   if (is_affinity_sessions (s))
 393     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 394                          s->in2out.protocol, s->out2in.port);
 395   l_addr = &s->out2in.addr;
 396   r_addr = &s->ext_host_addr;
 397   fib_index = s->out2in.fib_index;
 398   if (snat_is_unk_proto_session (s))
 399     {
 400       proto = s->in2out.port;
 401       r_port = 0;
 402       l_port = 0;
 403     }
 404   else
 405     {
 406       proto = nat_proto_to_ip_proto (s->in2out.protocol);
 407       l_port = s->out2in.port;
 408       r_port = s->ext_host_port;
 409     }
 410   make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 411               &ed_kv);
 412
 413   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0)))
 414     nat_elog_warn ("out2in_ed key del failed");
 415
 416   l_addr = &s->in2out.addr;
 417   fib_index = s->in2out.fib_index;
 418
 419   if (!snat_is_unk_proto_session (s))
 420     l_port = s->in2out.port;
 421
 422   if (is_twice_nat_session (s))
 423     {
 424       r_addr = &s->ext_host_nat_addr;
 425       r_port = s->ext_host_nat_port;
 426     }
 427   make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 428               &ed_kv);
 429
 430   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 431     nat_elog_warn ("in2out_ed key del failed");
 432
 433   if (!is_ha)
 434     {
 435       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 436                              &s->in2out.addr, s->in2out.port,
 437                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 438                              &s->out2in.addr, s->out2in.port,
 439                              &s->ext_host_addr, s->ext_host_port,
 440                              s->in2out.protocol, is_twice_nat_session (s));
 441     }
 442
 443   if (snat_is_unk_proto_session (s))
 444     return;
 445
 446   // is this correct ?
 447   if (!is_ha)
 448     {
 449       snat_ipfix_logging_nat44_ses_delete (thread_index,
 450                                            s->in2out.addr.as_u32,
 451                                            s->out2in.addr.as_u32,
 452                                            s->in2out.protocol,
 453                                            s->in2out.port,
 454                                            s->out2in.port,
 455                                            s->in2out.fib_index);
 456       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 457                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 458                    thread_index);
 459     }
 460
 461   /* Twice NAT address and port for external host */
 462   if (is_twice_nat_session (s))
 463     {
 464       key.protocol = s->in2out.protocol;
 465       key.port = s->ext_host_nat_port;
 466       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 467       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 468                                           thread_index, &key);
 469     }
 470
 471   if (snat_is_session_static (s))
 472     return;
 473
 474   // should be called for every dynamic session
 475   snat_free_outside_address_and_port (sm->addresses, thread_index,
 476                                       &s->out2in);
 477 }
 478
 479
 480 snat_user_t *
 481 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 482                         u32 thread_index)
 483 {
 484   snat_user_t *u = 0;
 485   snat_user_key_t user_key;
 486   clib_bihash_kv_8_8_t kv, value;
 487   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 488   dlist_elt_t *per_user_list_head_elt;
 489
 490   user_key.addr.as_u32 = addr->as_u32;
 491   user_key.fib_index = fib_index;
 492   kv.key = user_key.as_u64;
 493
 494   /* Ever heard of the "user" = src ip4 address before? */
 495   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 496     {
 497       /* no, make a new one */
 498       pool_get (tsm->users, u);
 499       clib_memset (u, 0, sizeof (*u));
 500
 501       u->addr.as_u32 = addr->as_u32;
 502       u->fib_index = fib_index;
 503
 504       pool_get (tsm->list_pool, per_user_list_head_elt);
 505
 506       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 507         tsm->list_pool;
 508
 509       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 510
 511       kv.value = u - tsm->users;
 512
 513       /* add user */
 514       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 515         {
 516           nat_elog_warn ("user_hash key add failed");
 517           nat44_delete_user_with_no_session (sm, u, thread_index);
 518           return NULL;
 519         }
 520
 521       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 522                                pool_elts (tsm->users));
 523     }
 524   else
 525     {
 526       u = pool_elt_at_index (tsm->users, value.value);
 527     }
 528
 529   return u;
 530 }
 531
 532 snat_session_t *
 533 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 534                               u32 thread_index, f64 now)
 535 {
 536   snat_session_t *s;
 537   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 538   u32 oldest_per_user_translation_list_index, session_index;
 539   dlist_elt_t *oldest_per_user_translation_list_elt;
 540   dlist_elt_t *per_user_translation_list_elt;
 541
 542   /* Over quota? Recycle the least recently used translation */
 543   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 544     {
 545       oldest_per_user_translation_list_index =
 546         clib_dlist_remove_head (tsm->list_pool,
 547                                 u->sessions_per_user_list_head_index);
 548
 549       ASSERT (oldest_per_user_translation_list_index != ~0);
 550
 551       /* Add it back to the end of the LRU list */
 552       clib_dlist_addtail (tsm->list_pool,
 553                           u->sessions_per_user_list_head_index,
 554                           oldest_per_user_translation_list_index);
 555       /* Get the list element */
 556       oldest_per_user_translation_list_elt =
 557         pool_elt_at_index (tsm->list_pool,
 558                            oldest_per_user_translation_list_index);
 559
 560       /* Get the session index from the list element */
 561       session_index = oldest_per_user_translation_list_elt->value;
 562
 563       /* Get the session */
 564       s = pool_elt_at_index (tsm->sessions, session_index);
 565       nat_free_session_data (sm, s, thread_index, 0);
 566       if (snat_is_session_static (s))
 567         u->nstaticsessions--;
 568       else
 569         u->nsessions--;
 570       s->flags = 0;
 571       s->total_bytes = 0;
 572       s->total_pkts = 0;
 573       s->state = 0;
 574       s->ext_host_addr.as_u32 = 0;
 575       s->ext_host_port = 0;
 576       s->ext_host_nat_addr.as_u32 = 0;
 577       s->ext_host_nat_port = 0;
 578     }
 579   else
 580     {
 581       pool_get (tsm->sessions, s);
 582       clib_memset (s, 0, sizeof (*s));
 583
 584       /* Create list elts */
 585       pool_get (tsm->list_pool, per_user_translation_list_elt);
 586       clib_dlist_init (tsm->list_pool,
 587                        per_user_translation_list_elt - tsm->list_pool);
 588
 589       per_user_translation_list_elt->value = s - tsm->sessions;
 590       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 591       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 592
 593       clib_dlist_addtail (tsm->list_pool,
 594                           s->per_user_list_head_index,
 595                           per_user_translation_list_elt - tsm->list_pool);
 596
 597       s->user_index = u - tsm->users;
 598       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 599                                pool_elts (tsm->sessions));
 600     }
 601
 602   s->ha_last_refreshed = now;
 603
 604   return s;
 605 }
 606
 607 void
 608 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 609                           int is_add)
 610 {
 611   fib_prefix_t prefix = {
 612     .fp_len = p_len,
 613     .fp_proto = FIB_PROTOCOL_IP4,
 614     .fp_addr = {
 615                 .ip4.as_u32 = addr->as_u32,
 616                 },
 617   };
 618   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 619
 620   if (is_add)
 621     fib_table_entry_update_one_path (fib_index,
 622                                      &prefix,
 623                                      nat_fib_src_low,
 624                                      (FIB_ENTRY_FLAG_CONNECTED |
 625                                       FIB_ENTRY_FLAG_LOCAL |
 626                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 627                                      DPO_PROTO_IP4,
 628                                      NULL,
 629                                      sw_if_index,
 630                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 631   else
 632     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 633 }
 634
 635 int
 636 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 637                   u8 twice_nat)
 638 {
 639   snat_address_t *ap;
 640   snat_interface_t *i;
 641   vlib_thread_main_t *tm = vlib_get_thread_main ();
 642
 643   if (twice_nat && !sm->endpoint_dependent)
 644     return VNET_API_ERROR_FEATURE_DISABLED;
 645
 646   /* Check if address already exists */
 647   /* *INDENT-OFF* */
 648   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 649     {
 650       if (ap->addr.as_u32 == addr->as_u32)
 651         return VNET_API_ERROR_VALUE_EXIST;
 652     }
 653   /* *INDENT-ON* */
 654
 655   if (twice_nat)
 656     vec_add2 (sm->twice_nat_addresses, ap, 1);
 657   else
 658     vec_add2 (sm->addresses, ap, 1);
 659
 660   ap->addr = *addr;
 661   if (vrf_id != ~0)
 662     ap->fib_index =
 663       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 664                                          nat_fib_src_low);
 665   else
 666     ap->fib_index = ~0;
 667 #define _(N, i, n, s) \
 668   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 669   ap->busy_##n##_ports = 0; \
 670   ap->busy_##n##_ports_per_thread = 0;\
 671   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 672   foreach_nat_protocol
 673 #undef _
 674     if (twice_nat)
 675     return 0;
 676
 677   /* Add external address to FIB */
 678   /* *INDENT-OFF* */
 679   pool_foreach (i, sm->interfaces,
 680   ({
 681     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 682       continue;
 683
 684     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 685     break;
 686   }));
 687   pool_foreach (i, sm->output_feature_interfaces,
 688   ({
 689     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 690       continue;
 691
 692     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 693     break;
 694   }));
 695   /* *INDENT-ON* */
 696
 697   return 0;
 698 }
 699
 700 static int
 701 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 702 {
 703   snat_static_mapping_t *m;
 704   /* *INDENT-OFF* */
 705   pool_foreach (m, sm->static_mappings,
 706   ({
 707       if (is_addr_only_static_mapping (m) ||
 708           is_out2in_only_static_mapping (m) ||
 709           is_identity_static_mapping (m))
 710         continue;
 711       if (m->external_addr.as_u32 == addr.as_u32)
 712         return 1;
 713   }));
 714   /* *INDENT-ON* */
 715
 716   return 0;
 717 }
 718
 719 static void
 720 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 721                                        ip4_address_t l_addr,
 722                                        u16 l_port,
 723                                        u32 sw_if_index,
 724                                        u16 e_port,
 725                                        u32 vrf_id,
 726                                        nat_protocol_t proto,
 727                                        int addr_only, int is_add, u8 * tag,
 728                                        int twice_nat, int out2in_only,
 729                                        int identity_nat)
 730 {
 731   snat_static_map_resolve_t *rp;
 732
 733   vec_add2 (sm->to_resolve, rp, 1);
 734   rp->l_addr.as_u32 = l_addr.as_u32;
 735   rp->l_port = l_port;
 736   rp->sw_if_index = sw_if_index;
 737   rp->e_port = e_port;
 738   rp->vrf_id = vrf_id;
 739   rp->proto = proto;
 740   rp->addr_only = addr_only;
 741   rp->is_add = is_add;
 742   rp->twice_nat = twice_nat;
 743   rp->out2in_only = out2in_only;
 744   rp->identity_nat = identity_nat;
 745   rp->tag = vec_dup (tag);
 746 }
 747
 748 static u32
 749 get_thread_idx_by_port (u16 e_port)
 750 {
 751   snat_main_t *sm = &snat_main;
 752   u32 thread_idx = sm->num_workers;
 753   if (sm->num_workers > 1)
 754     {
 755       thread_idx =
 756         sm->first_worker_index +
 757         sm->workers[(e_port - 1024) / sm->port_per_thread];
 758     }
 759   return thread_idx;
 760 }
 761
 762 void
 763 snat_static_mapping_del_sessions (snat_main_t * sm,
 764                                   snat_main_per_thread_data_t * tsm,
 765                                   snat_user_key_t u_key, int addr_only,
 766                                   ip4_address_t e_addr, u16 e_port)
 767 {
 768   clib_bihash_kv_8_8_t kv, value;
 769   kv.key = u_key.as_u64;
 770   u64 user_index;
 771   dlist_elt_t *head, *elt;
 772   snat_user_t *u;
 773   snat_session_t *s;
 774   u32 elt_index, head_index, ses_index;
 775   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 776     {
 777       user_index = value.value;
 778       u = pool_elt_at_index (tsm->users, user_index);
 779       if (u->nstaticsessions)
 780         {
 781           head_index = u->sessions_per_user_list_head_index;
 782           head = pool_elt_at_index (tsm->list_pool, head_index);
 783           elt_index = head->next;
 784           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 785           ses_index = elt->value;
 786           while (ses_index != ~0)
 787             {
 788               s = pool_elt_at_index (tsm->sessions, ses_index);
 789               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 790               ses_index = elt->value;
 791
 792               if (!addr_only)
 793                 {
 794                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 795                       (clib_net_to_host_u16 (s->out2in.port) != e_port))
 796                     continue;
 797                 }
 798
 799               if (is_lb_session (s))
 800                 continue;
 801
 802               if (!snat_is_session_static (s))
 803                 continue;
 804
 805               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 806               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 807
 808               if (!addr_only)
 809                 break;
 810             }
 811         }
 812     }
 813 }
 814
 815 void
 816 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 817                                      snat_main_per_thread_data_t * tsm,
 818                                      ip4_address_t l_addr,
 819                                      u16 l_port,
 820                                      u8 protocol,
 821                                      u32 fib_index, int addr_only,
 822                                      ip4_address_t e_addr, u16 e_port)
 823 {
 824   snat_session_t *s;
 825   u32 *indexes_to_free = NULL;
 826   /* *INDENT-OFF* */
 827   pool_foreach (s, tsm->sessions, {
 828     if (s->in2out.fib_index != fib_index ||
 829         s->in2out.addr.as_u32 != l_addr.as_u32)
 830       {
 831         continue;
 832       }
 833     if (!addr_only)
 834       {
 835         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 836             (clib_net_to_host_u16 (s->out2in.port) != e_port) ||
 837             clib_net_to_host_u16 (s->in2out.port) != l_port ||
 838             s->in2out.protocol != protocol)
 839           continue;
 840       }
 841
 842     if (is_lb_session (s))
 843       continue;
 844     if (!snat_is_session_static (s))
 845       continue;
 846     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 847     vec_add1 (indexes_to_free, s - tsm->sessions);
 848     if (!addr_only)
 849       break;
 850   });
 851   /* *INDENT-ON* */
 852   u32 *ses_index;
 853   vec_foreach (ses_index, indexes_to_free)
 854   {
 855     s = pool_elt_at_index (tsm->sessions, *ses_index);
 856     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 857   }
 858   vec_free (indexes_to_free);
 859 }
 860
 861 int
 862 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 863                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 864                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 865                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 866                          u8 identity_nat)
 867 {
 868   snat_main_t *sm = &snat_main;
 869   snat_static_mapping_t *m;
 870   snat_session_key_t m_key;
 871   clib_bihash_kv_8_8_t kv, value;
 872   snat_address_t *a = 0;
 873   u32 fib_index = ~0;
 874   snat_interface_t *interface;
 875   int i;
 876   snat_main_per_thread_data_t *tsm;
 877   snat_user_key_t u_key;
 878   snat_user_t *u;
 879   dlist_elt_t *head, *elt;
 880   u32 elt_index, head_index;
 881   u32 ses_index;
 882   u64 user_index;
 883   snat_session_t *s;
 884   snat_static_map_resolve_t *rp, *rp_match = 0;
 885   nat44_lb_addr_port_t *local;
 886   u32 find = ~0;
 887
 888   if (!sm->endpoint_dependent)
 889     {
 890       if (twice_nat || out2in_only)
 891         return VNET_API_ERROR_FEATURE_DISABLED;
 892     }
 893
 894   /* If the external address is a specific interface address */
 895   if (sw_if_index != ~0)
 896     {
 897       ip4_address_t *first_int_addr;
 898
 899       for (i = 0; i < vec_len (sm->to_resolve); i++)
 900         {
 901           rp = sm->to_resolve + i;
 902           if (rp->sw_if_index != sw_if_index ||
 903               rp->l_addr.as_u32 != l_addr.as_u32 ||
 904               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 905             continue;
 906
 907           if (!addr_only)
 908             {
 909               if ((rp->l_port != l_port && rp->e_port != e_port)
 910                   || rp->proto != proto)
 911                 continue;
 912             }
 913
 914           rp_match = rp;
 915           break;
 916         }
 917
 918       /* Might be already set... */
 919       first_int_addr = ip4_interface_first_address
 920         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 921
 922       if (is_add)
 923         {
 924           if (rp_match)
 925             return VNET_API_ERROR_VALUE_EXIST;
 926
 927           snat_add_static_mapping_when_resolved
 928             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 929              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 930
 931           /* DHCP resolution required? */
 932           if (first_int_addr == 0)
 933             {
 934               return 0;
 935             }
 936           else
 937             {
 938               e_addr.as_u32 = first_int_addr->as_u32;
 939               /* Identity mapping? */
 940               if (l_addr.as_u32 == 0)
 941                 l_addr.as_u32 = e_addr.as_u32;
 942             }
 943         }
 944       else
 945         {
 946           if (!rp_match)
 947             return VNET_API_ERROR_NO_SUCH_ENTRY;
 948
 949           vec_del1 (sm->to_resolve, i);
 950
 951           if (first_int_addr)
 952             {
 953               e_addr.as_u32 = first_int_addr->as_u32;
 954               /* Identity mapping? */
 955               if (l_addr.as_u32 == 0)
 956                 l_addr.as_u32 = e_addr.as_u32;
 957             }
 958           else
 959             return 0;
 960         }
 961     }
 962
 963   m_key.addr = e_addr;
 964   m_key.port = addr_only ? 0 : e_port;
 965   m_key.protocol = addr_only ? 0 : proto;
 966   m_key.fib_index = 0;
 967   kv.key = m_key.as_u64;
 968   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 969     m = 0;
 970   else
 971     m = pool_elt_at_index (sm->static_mappings, value.value);
 972
 973   if (is_add)
 974     {
 975       if (m)
 976         {
 977           if (is_identity_static_mapping (m))
 978             {
 979               /* *INDENT-OFF* */
 980               pool_foreach (local, m->locals,
 981               ({
 982                 if (local->vrf_id == vrf_id)
 983                   return VNET_API_ERROR_VALUE_EXIST;
 984               }));
 985               /* *INDENT-ON* */
 986               pool_get (m->locals, local);
 987               local->vrf_id = vrf_id;
 988               local->fib_index =
 989                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 990                                                    nat_fib_src_low);
 991               m_key.addr = m->local_addr;
 992               m_key.port = m->local_port;
 993               m_key.protocol = m->proto;
 994               m_key.fib_index = local->fib_index;
 995               kv.key = m_key.as_u64;
 996               kv.value = m - sm->static_mappings;
 997               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 998               return 0;
 999             }
1000           else
1001             return VNET_API_ERROR_VALUE_EXIST;
1002         }
1003
1004       if (twice_nat && addr_only)
1005         return VNET_API_ERROR_UNSUPPORTED;
1006
1007       /* Convert VRF id to FIB index */
1008       if (vrf_id != ~0)
1009         fib_index =
1010           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1011                                              nat_fib_src_low);
1012       /* If not specified use inside VRF id from SNAT plugin startup config */
1013       else
1014         {
1015           fib_index = sm->inside_fib_index;
1016           vrf_id = sm->inside_vrf_id;
1017           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1018         }
1019
1020       if (!(out2in_only || identity_nat))
1021         {
1022           m_key.addr = l_addr;
1023           m_key.port = addr_only ? 0 : l_port;
1024           m_key.protocol = addr_only ? 0 : proto;
1025           m_key.fib_index = fib_index;
1026           kv.key = m_key.as_u64;
1027           if (!clib_bihash_search_8_8
1028               (&sm->static_mapping_by_local, &kv, &value))
1029             return VNET_API_ERROR_VALUE_EXIST;
1030         }
1031
1032       /* Find external address in allocated addresses and reserve port for
1033          address and port pair mapping when dynamic translations enabled */
1034       if (!(addr_only || sm->static_mapping_only || out2in_only))
1035         {
1036           for (i = 0; i < vec_len (sm->addresses); i++)
1037             {
1038               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1039                 {
1040                   a = sm->addresses + i;
1041                   /* External port must be unused */
1042                   switch (proto)
1043                     {
1044 #define _(N, j, n, s) \
1045                     case NAT_PROTOCOL_##N: \
1046                       if (a->busy_##n##_port_refcounts[e_port]) \
1047                         return VNET_API_ERROR_INVALID_VALUE; \
1048                       ++a->busy_##n##_port_refcounts[e_port]; \
1049                       if (e_port > 1024) \
1050                         { \
1051                           a->busy_##n##_ports++; \
1052                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1053                         } \
1054                       break;
1055                       foreach_nat_protocol
1056 #undef _
1057                     default:
1058                       nat_elog_info ("unknown protocol");
1059                       return VNET_API_ERROR_INVALID_VALUE_2;
1060                     }
1061                   break;
1062                 }
1063             }
1064           /* External address must be allocated */
1065           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1066             {
1067               if (sw_if_index != ~0)
1068                 {
1069                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1070                     {
1071                       rp = sm->to_resolve + i;
1072                       if (rp->addr_only)
1073                         continue;
1074                       if (rp->sw_if_index != sw_if_index &&
1075                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1076                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1077                           rp->e_port != e_port && rp->proto != proto)
1078                         continue;
1079
1080                       vec_del1 (sm->to_resolve, i);
1081                       break;
1082                     }
1083                 }
1084               return VNET_API_ERROR_NO_SUCH_ENTRY;
1085             }
1086         }
1087
1088       pool_get (sm->static_mappings, m);
1089       clib_memset (m, 0, sizeof (*m));
1090       m->tag = vec_dup (tag);
1091       m->local_addr = l_addr;
1092       m->external_addr = e_addr;
1093       m->twice_nat = twice_nat;
1094       if (out2in_only)
1095         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1096       if (addr_only)
1097         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1098       if (identity_nat)
1099         {
1100           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1101           pool_get (m->locals, local);
1102           local->vrf_id = vrf_id;
1103           local->fib_index = fib_index;
1104         }
1105       else
1106         {
1107           m->vrf_id = vrf_id;
1108           m->fib_index = fib_index;
1109         }
1110       if (!addr_only)
1111         {
1112           m->local_port = l_port;
1113           m->external_port = e_port;
1114           m->proto = proto;
1115         }
1116
1117       if (sm->num_workers > 1)
1118         {
1119           ip4_header_t ip = {
1120             .src_address = m->local_addr,
1121           };
1122           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1123           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1124         }
1125       else
1126         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1127
1128       m_key.addr = m->local_addr;
1129       m_key.port = m->local_port;
1130       m_key.protocol = m->proto;
1131       m_key.fib_index = fib_index;
1132       kv.key = m_key.as_u64;
1133       kv.value = m - sm->static_mappings;
1134       if (!out2in_only)
1135         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1136
1137       m_key.addr = m->external_addr;
1138       m_key.port = m->external_port;
1139       m_key.fib_index = 0;
1140       kv.key = m_key.as_u64;
1141       kv.value = m - sm->static_mappings;
1142       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1143
1144       /* Delete dynamic sessions matching local address (+ local port) */
1145       if (!(sm->static_mapping_only))
1146         {
1147           u_key.addr = m->local_addr;
1148           u_key.fib_index = m->fib_index;
1149           kv.key = u_key.as_u64;
1150           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1151             {
1152               user_index = value.value;
1153               u = pool_elt_at_index (tsm->users, user_index);
1154               if (u->nsessions)
1155                 {
1156                   head_index = u->sessions_per_user_list_head_index;
1157                   head = pool_elt_at_index (tsm->list_pool, head_index);
1158                   elt_index = head->next;
1159                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1160                   ses_index = elt->value;
1161                   while (ses_index != ~0)
1162                     {
1163                       s = pool_elt_at_index (tsm->sessions, ses_index);
1164                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1165                       ses_index = elt->value;
1166
1167                       if (snat_is_session_static (s))
1168                         continue;
1169
1170                       if (!addr_only
1171                           && (clib_net_to_host_u16 (s->in2out.port) !=
1172                               m->local_port))
1173                         continue;
1174
1175                       nat_free_session_data (sm, s,
1176                                              tsm - sm->per_thread_data, 0);
1177                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1178
1179                       if (!addr_only && !sm->endpoint_dependent)
1180                         break;
1181                     }
1182                 }
1183             }
1184         }
1185     }
1186   else
1187     {
1188       if (!m)
1189         {
1190           if (sw_if_index != ~0)
1191             return 0;
1192           else
1193             return VNET_API_ERROR_NO_SUCH_ENTRY;
1194         }
1195
1196       if (identity_nat)
1197         {
1198           if (vrf_id == ~0)
1199             vrf_id = sm->inside_vrf_id;
1200
1201           /* *INDENT-OFF* */
1202           pool_foreach (local, m->locals,
1203           ({
1204             if (local->vrf_id == vrf_id)
1205               find = local - m->locals;
1206           }));
1207           /* *INDENT-ON* */
1208           if (find == ~0)
1209             return VNET_API_ERROR_NO_SUCH_ENTRY;
1210
1211           local = pool_elt_at_index (m->locals, find);
1212           fib_index = local->fib_index;
1213           pool_put (m->locals, local);
1214         }
1215       else
1216         fib_index = m->fib_index;
1217
1218       /* Free external address port */
1219       if (!(addr_only || sm->static_mapping_only || out2in_only))
1220         {
1221           for (i = 0; i < vec_len (sm->addresses); i++)
1222             {
1223               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1224                 {
1225                   a = sm->addresses + i;
1226                   switch (proto)
1227                     {
1228 #define _(N, j, n, s) \
1229                     case NAT_PROTOCOL_##N: \
1230                       --a->busy_##n##_port_refcounts[e_port]; \
1231                       if (e_port > 1024) \
1232                         { \
1233                           a->busy_##n##_ports--; \
1234                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1235                         } \
1236                       break;
1237                       foreach_nat_protocol
1238 #undef _
1239                     default:
1240                       nat_elog_info ("unknown protocol");
1241                       return VNET_API_ERROR_INVALID_VALUE_2;
1242                     }
1243                   break;
1244                 }
1245             }
1246         }
1247
1248       if (sm->num_workers > 1)
1249         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1250       else
1251         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1252
1253       m_key.addr = m->local_addr;
1254       m_key.port = m->local_port;
1255       m_key.protocol = m->proto;
1256       m_key.fib_index = fib_index;
1257       kv.key = m_key.as_u64;
1258       if (!out2in_only)
1259         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1260
1261       /* Delete session(s) for static mapping if exist */
1262       if (!(sm->static_mapping_only) ||
1263           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1264         {
1265           if (sm->endpoint_dependent)
1266             {
1267               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1268                                                    m->local_port, m->proto,
1269                                                    fib_index, addr_only,
1270                                                    e_addr, e_port);
1271             }
1272           else
1273             {
1274               u_key.addr = m->local_addr;
1275               u_key.fib_index = fib_index;
1276               kv.key = u_key.as_u64;
1277               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1278                                                 e_addr, e_port);
1279             }
1280         }
1281
1282       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1283       if (pool_elts (m->locals))
1284         return 0;
1285
1286       m_key.addr = m->external_addr;
1287       m_key.port = m->external_port;
1288       m_key.fib_index = 0;
1289       kv.key = m_key.as_u64;
1290       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1291
1292       vec_free (m->tag);
1293       vec_free (m->workers);
1294       /* Delete static mapping from pool */
1295       pool_put (sm->static_mappings, m);
1296     }
1297
1298   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1299     return 0;
1300
1301   /* Add/delete external address to FIB */
1302   /* *INDENT-OFF* */
1303   pool_foreach (interface, sm->interfaces,
1304   ({
1305     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1306       continue;
1307
1308     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1309     break;
1310   }));
1311   pool_foreach (interface, sm->output_feature_interfaces,
1312   ({
1313     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1314       continue;
1315
1316     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1317     break;
1318   }));
1319   /* *INDENT-ON* */
1320
1321   return 0;
1322 }
1323
1324 int
1325 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1326                                  nat_protocol_t proto,
1327                                  nat44_lb_addr_port_t * locals, u8 is_add,
1328                                  twice_nat_type_t twice_nat, u8 out2in_only,
1329                                  u8 * tag, u32 affinity)
1330 {
1331   snat_main_t *sm = &snat_main;
1332   snat_static_mapping_t *m;
1333   snat_session_key_t m_key;
1334   clib_bihash_kv_8_8_t kv, value;
1335   snat_address_t *a = 0;
1336   int i;
1337   nat44_lb_addr_port_t *local;
1338   snat_main_per_thread_data_t *tsm;
1339   snat_session_t *s;
1340   uword *bitmap = 0;
1341
1342   if (!sm->endpoint_dependent)
1343     return VNET_API_ERROR_FEATURE_DISABLED;
1344
1345   m_key.addr = e_addr;
1346   m_key.port = e_port;
1347   m_key.protocol = proto;
1348   m_key.fib_index = 0;
1349   kv.key = m_key.as_u64;
1350   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1351     m = 0;
1352   else
1353     m = pool_elt_at_index (sm->static_mappings, value.value);
1354
1355   if (is_add)
1356     {
1357       if (m)
1358         return VNET_API_ERROR_VALUE_EXIST;
1359
1360       if (vec_len (locals) < 2)
1361         return VNET_API_ERROR_INVALID_VALUE;
1362
1363       /* Find external address in allocated addresses and reserve port for
1364          address and port pair mapping when dynamic translations enabled */
1365       if (!(sm->static_mapping_only || out2in_only))
1366         {
1367           for (i = 0; i < vec_len (sm->addresses); i++)
1368             {
1369               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1370                 {
1371                   a = sm->addresses + i;
1372                   /* External port must be unused */
1373                   switch (proto)
1374                     {
1375 #define _(N, j, n, s) \
1376                     case NAT_PROTOCOL_##N: \
1377                       if (a->busy_##n##_port_refcounts[e_port]) \
1378                         return VNET_API_ERROR_INVALID_VALUE; \
1379                       ++a->busy_##n##_port_refcounts[e_port]; \
1380                       if (e_port > 1024) \
1381                         { \
1382                           a->busy_##n##_ports++; \
1383                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1384                         } \
1385                       break;
1386                       foreach_nat_protocol
1387 #undef _
1388                     default:
1389                       nat_elog_info ("unknown protocol");
1390                       return VNET_API_ERROR_INVALID_VALUE_2;
1391                     }
1392                   break;
1393                 }
1394             }
1395           /* External address must be allocated */
1396           if (!a)
1397             return VNET_API_ERROR_NO_SUCH_ENTRY;
1398         }
1399
1400       pool_get (sm->static_mappings, m);
1401       clib_memset (m, 0, sizeof (*m));
1402       m->tag = vec_dup (tag);
1403       m->external_addr = e_addr;
1404       m->external_port = e_port;
1405       m->proto = proto;
1406       m->twice_nat = twice_nat;
1407       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1408       if (out2in_only)
1409         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1410       m->affinity = affinity;
1411
1412       if (affinity)
1413         m->affinity_per_service_list_head_index =
1414           nat_affinity_get_per_service_list_head_index ();
1415       else
1416         m->affinity_per_service_list_head_index = ~0;
1417
1418       m_key.addr = m->external_addr;
1419       m_key.port = m->external_port;
1420       m_key.protocol = m->proto;
1421       m_key.fib_index = 0;
1422       kv.key = m_key.as_u64;
1423       kv.value = m - sm->static_mappings;
1424       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1425         {
1426           nat_elog_err ("static_mapping_by_external key add failed");
1427           return VNET_API_ERROR_UNSPECIFIED;
1428         }
1429
1430       m_key.fib_index = m->fib_index;
1431       for (i = 0; i < vec_len (locals); i++)
1432         {
1433           locals[i].fib_index =
1434             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1435                                                locals[i].vrf_id,
1436                                                nat_fib_src_low);
1437           m_key.addr = locals[i].addr;
1438           m_key.fib_index = locals[i].fib_index;
1439           if (!out2in_only)
1440             {
1441               m_key.port = locals[i].port;
1442               kv.key = m_key.as_u64;
1443               kv.value = m - sm->static_mappings;
1444               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1445             }
1446           locals[i].prefix = (i == 0) ? locals[i].probability :
1447             (locals[i - 1].prefix + locals[i].probability);
1448           pool_get (m->locals, local);
1449           *local = locals[i];
1450           if (sm->num_workers > 1)
1451             {
1452               ip4_header_t ip = {
1453                 .src_address = locals[i].addr,
1454               };
1455               bitmap =
1456                 clib_bitmap_set (bitmap,
1457                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1458                                  1);
1459             }
1460         }
1461
1462       /* Assign workers */
1463       if (sm->num_workers > 1)
1464         {
1465           /* *INDENT-OFF* */
1466           clib_bitmap_foreach (i, bitmap,
1467             ({
1468                vec_add1(m->workers, i);
1469             }));
1470           /* *INDENT-ON* */
1471         }
1472     }
1473   else
1474     {
1475       if (!m)
1476         return VNET_API_ERROR_NO_SUCH_ENTRY;
1477
1478       if (!is_lb_static_mapping (m))
1479         return VNET_API_ERROR_INVALID_VALUE;
1480
1481       /* Free external address port */
1482       if (!(sm->static_mapping_only || out2in_only))
1483         {
1484           for (i = 0; i < vec_len (sm->addresses); i++)
1485             {
1486               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1487                 {
1488                   a = sm->addresses + i;
1489                   switch (proto)
1490                     {
1491 #define _(N, j, n, s) \
1492                     case NAT_PROTOCOL_##N: \
1493                       --a->busy_##n##_port_refcounts[e_port]; \
1494                       if (e_port > 1024) \
1495                         { \
1496                           a->busy_##n##_ports--; \
1497                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1498                         } \
1499                       break;
1500                       foreach_nat_protocol
1501 #undef _
1502                     default:
1503                       nat_elog_info ("unknown protocol");
1504                       return VNET_API_ERROR_INVALID_VALUE_2;
1505                     }
1506                   break;
1507                 }
1508             }
1509         }
1510
1511       m_key.addr = m->external_addr;
1512       m_key.port = m->external_port;
1513       m_key.protocol = m->proto;
1514       m_key.fib_index = 0;
1515       kv.key = m_key.as_u64;
1516       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1517         {
1518           nat_elog_err ("static_mapping_by_external key del failed");
1519           return VNET_API_ERROR_UNSPECIFIED;
1520         }
1521
1522       /* *INDENT-OFF* */
1523       pool_foreach (local, m->locals,
1524       ({
1525           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1526                             nat_fib_src_low);
1527           m_key.addr = local->addr;
1528           if (!out2in_only)
1529             {
1530               m_key.port = local->port;
1531               m_key.fib_index = local->fib_index;
1532               kv.key = m_key.as_u64;
1533               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1534                 {
1535                   nat_elog_err ("static_mapping_by_local key del failed");
1536                   return VNET_API_ERROR_UNSPECIFIED;
1537                 }
1538             }
1539
1540           if (sm->num_workers > 1)
1541             {
1542               ip4_header_t ip = {
1543                 .src_address = local->addr,
1544               };
1545               tsm = vec_elt_at_index (sm->per_thread_data,
1546                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1547             }
1548           else
1549             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1550
1551           /* Delete sessions */
1552           pool_foreach (s, tsm->sessions, {
1553             if (!(is_lb_session (s)))
1554               continue;
1555
1556             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1557                 (clib_net_to_host_u16 (s->in2out.port) != local->port))
1558               continue;
1559
1560             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1561             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1562           });
1563       }));
1564       /* *INDENT-ON* */
1565       if (m->affinity)
1566         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1567       pool_free (m->locals);
1568       vec_free (m->tag);
1569       vec_free (m->workers);
1570
1571       pool_put (sm->static_mappings, m);
1572     }
1573
1574   return 0;
1575 }
1576
1577 int
1578 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1579                                        ip4_address_t l_addr, u16 l_port,
1580                                        nat_protocol_t proto, u32 vrf_id,
1581                                        u8 probability, u8 is_add)
1582 {
1583   snat_main_t *sm = &snat_main;
1584   snat_static_mapping_t *m = 0;
1585   snat_session_key_t m_key;
1586   clib_bihash_kv_8_8_t kv, value;
1587   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1588   snat_main_per_thread_data_t *tsm;
1589   snat_session_t *s;
1590   u32 *locals = 0;
1591   uword *bitmap = 0;
1592   int i;
1593
1594   if (!sm->endpoint_dependent)
1595     return VNET_API_ERROR_FEATURE_DISABLED;
1596
1597   m_key.addr = e_addr;
1598   m_key.port = e_port;
1599   m_key.protocol = proto;
1600   m_key.fib_index = 0;
1601   kv.key = m_key.as_u64;
1602   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1603     m = pool_elt_at_index (sm->static_mappings, value.value);
1604
1605   if (!m)
1606     return VNET_API_ERROR_NO_SUCH_ENTRY;
1607
1608   if (!is_lb_static_mapping (m))
1609     return VNET_API_ERROR_INVALID_VALUE;
1610
1611   /* *INDENT-OFF* */
1612   pool_foreach (local, m->locals,
1613   ({
1614     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1615         (local->vrf_id == vrf_id))
1616       {
1617         match_local = local;
1618         break;
1619       }
1620   }));
1621   /* *INDENT-ON* */
1622
1623   if (is_add)
1624     {
1625       if (match_local)
1626         return VNET_API_ERROR_VALUE_EXIST;
1627
1628       pool_get (m->locals, local);
1629       clib_memset (local, 0, sizeof (*local));
1630       local->addr.as_u32 = l_addr.as_u32;
1631       local->port = l_port;
1632       local->probability = probability;
1633       local->vrf_id = vrf_id;
1634       local->fib_index =
1635         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1636                                            nat_fib_src_low);
1637
1638       if (!is_out2in_only_static_mapping (m))
1639         {
1640           m_key.addr = l_addr;
1641           m_key.port = l_port;
1642           m_key.fib_index = local->fib_index;
1643           kv.key = m_key.as_u64;
1644           kv.value = m - sm->static_mappings;
1645           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1646             nat_elog_err ("static_mapping_by_local key add failed");
1647         }
1648     }
1649   else
1650     {
1651       if (!match_local)
1652         return VNET_API_ERROR_NO_SUCH_ENTRY;
1653
1654       if (pool_elts (m->locals) < 3)
1655         return VNET_API_ERROR_UNSPECIFIED;
1656
1657       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1658                         nat_fib_src_low);
1659
1660       if (!is_out2in_only_static_mapping (m))
1661         {
1662           m_key.addr = l_addr;
1663           m_key.port = l_port;
1664           m_key.fib_index = match_local->fib_index;
1665           kv.key = m_key.as_u64;
1666           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1667             nat_elog_err ("static_mapping_by_local key del failed");
1668         }
1669
1670       if (sm->num_workers > 1)
1671         {
1672           ip4_header_t ip = {
1673             .src_address = local->addr,
1674           };
1675           tsm = vec_elt_at_index (sm->per_thread_data,
1676                                   sm->worker_in2out_cb (&ip, m->fib_index,
1677                                                         0));
1678         }
1679       else
1680         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1681
1682       /* Delete sessions */
1683       /* *INDENT-OFF* */
1684       pool_foreach (s, tsm->sessions, {
1685         if (!(is_lb_session (s)))
1686           continue;
1687
1688         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1689             (clib_net_to_host_u16 (s->in2out.port) != match_local->port))
1690           continue;
1691
1692         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1693         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1694       });
1695       /* *INDENT-ON* */
1696
1697       pool_put (m->locals, match_local);
1698     }
1699
1700   vec_free (m->workers);
1701
1702   /* *INDENT-OFF* */
1703   pool_foreach (local, m->locals,
1704   ({
1705     vec_add1 (locals, local - m->locals);
1706     if (sm->num_workers > 1)
1707       {
1708         ip4_header_t ip;
1709         ip.src_address.as_u32 = local->addr.as_u32,
1710         bitmap = clib_bitmap_set (bitmap,
1711                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1712                                   1);
1713       }
1714   }));
1715   /* *INDENT-ON* */
1716
1717   ASSERT (vec_len (locals) > 1);
1718
1719   local = pool_elt_at_index (m->locals, locals[0]);
1720   local->prefix = local->probability;
1721   for (i = 1; i < vec_len (locals); i++)
1722     {
1723       local = pool_elt_at_index (m->locals, locals[i]);
1724       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1725       local->prefix = local->probability + prev_local->prefix;
1726     }
1727
1728   /* Assign workers */
1729   if (sm->num_workers > 1)
1730     {
1731       /* *INDENT-OFF* */
1732       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1733       /* *INDENT-ON* */
1734     }
1735
1736   return 0;
1737 }
1738
1739 int
1740 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1741                   u8 twice_nat)
1742 {
1743   snat_address_t *a = 0;
1744   snat_session_t *ses;
1745   u32 *ses_to_be_removed = 0, *ses_index;
1746   snat_main_per_thread_data_t *tsm;
1747   snat_static_mapping_t *m;
1748   snat_interface_t *interface;
1749   int i;
1750   snat_address_t *addresses =
1751     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1752
1753   /* Find SNAT address */
1754   for (i = 0; i < vec_len (addresses); i++)
1755     {
1756       if (addresses[i].addr.as_u32 == addr.as_u32)
1757         {
1758           a = addresses + i;
1759           break;
1760         }
1761     }
1762   if (!a)
1763     return VNET_API_ERROR_NO_SUCH_ENTRY;
1764
1765   if (delete_sm)
1766     {
1767       /* *INDENT-OFF* */
1768       pool_foreach (m, sm->static_mappings,
1769       ({
1770           if (m->external_addr.as_u32 == addr.as_u32)
1771             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1772                                             m->local_port, m->external_port,
1773                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1774                                             m->proto, 0, m->twice_nat,
1775                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1776       }));
1777       /* *INDENT-ON* */
1778     }
1779   else
1780     {
1781       /* Check if address is used in some static mapping */
1782       if (is_snat_address_used_in_static_mapping (sm, addr))
1783         {
1784           nat_elog_notice ("address used in static mapping");
1785           return VNET_API_ERROR_UNSPECIFIED;
1786         }
1787     }
1788
1789   if (a->fib_index != ~0)
1790     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1791
1792   /* Delete sessions using address */
1793   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1794     {
1795       /* *INDENT-OFF* */
1796       vec_foreach (tsm, sm->per_thread_data)
1797         {
1798           pool_foreach (ses, tsm->sessions, ({
1799             if (ses->out2in.addr.as_u32 == addr.as_u32)
1800               {
1801                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1802                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1803               }
1804           }));
1805
1806           if (sm->endpoint_dependent){
1807               vec_foreach (ses_index, ses_to_be_removed)
1808                 {
1809                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1810                   nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1811                 }
1812           }else{
1813               vec_foreach (ses_index, ses_to_be_removed)
1814                 {
1815                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1816                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1817                 }
1818           }
1819
1820           vec_free (ses_to_be_removed);
1821         }
1822       /* *INDENT-ON* */
1823     }
1824
1825 #define _(N, i, n, s) \
1826   vec_free (a->busy_##n##_ports_per_thread);
1827   foreach_nat_protocol
1828 #undef _
1829     if (twice_nat)
1830     {
1831       vec_del1 (sm->twice_nat_addresses, i);
1832       return 0;
1833     }
1834   else
1835     vec_del1 (sm->addresses, i);
1836
1837   /* Delete external address from FIB */
1838   /* *INDENT-OFF* */
1839   pool_foreach (interface, sm->interfaces,
1840   ({
1841     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1842       continue;
1843
1844     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1845     break;
1846   }));
1847   pool_foreach (interface, sm->output_feature_interfaces,
1848   ({
1849     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1850       continue;
1851
1852     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1853     break;
1854   }));
1855   /* *INDENT-ON* */
1856
1857   return 0;
1858 }
1859
1860 int
1861 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1862 {
1863   snat_main_t *sm = &snat_main;
1864   snat_interface_t *i;
1865   const char *feature_name, *del_feature_name;
1866   snat_address_t *ap;
1867   snat_static_mapping_t *m;
1868   snat_det_map_t *dm;
1869   nat_outside_fib_t *outside_fib;
1870   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1871                                                        sw_if_index);
1872
1873   if (sm->out2in_dpo && !is_inside)
1874     return VNET_API_ERROR_UNSUPPORTED;
1875
1876   /* *INDENT-OFF* */
1877   pool_foreach (i, sm->output_feature_interfaces,
1878   ({
1879     if (i->sw_if_index == sw_if_index)
1880       return VNET_API_ERROR_VALUE_EXIST;
1881   }));
1882   /* *INDENT-ON* */
1883
1884   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1885     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1886   else
1887     {
1888       if (sm->num_workers > 1 && !sm->deterministic)
1889         feature_name =
1890           is_inside ? "nat44-in2out-worker-handoff" :
1891           "nat44-out2in-worker-handoff";
1892       else if (sm->deterministic)
1893         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1894       else if (sm->endpoint_dependent)
1895         {
1896           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1897         }
1898       else
1899         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1900     }
1901
1902   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1903     sm->fq_in2out_index =
1904       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1905
1906   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1907     sm->fq_out2in_index =
1908       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1909
1910   if (!is_inside)
1911     {
1912       /* *INDENT-OFF* */
1913       vec_foreach (outside_fib, sm->outside_fibs)
1914         {
1915           if (outside_fib->fib_index == fib_index)
1916             {
1917               if (is_del)
1918                 {
1919                   outside_fib->refcount--;
1920                   if (!outside_fib->refcount)
1921                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1922                 }
1923               else
1924                 outside_fib->refcount++;
1925               goto feature_set;
1926             }
1927         }
1928       /* *INDENT-ON* */
1929       if (!is_del)
1930         {
1931           vec_add2 (sm->outside_fibs, outside_fib, 1);
1932           outside_fib->refcount = 1;
1933           outside_fib->fib_index = fib_index;
1934         }
1935     }
1936 feature_set:
1937   /* *INDENT-OFF* */
1938   pool_foreach (i, sm->interfaces,
1939   ({
1940     if (i->sw_if_index == sw_if_index)
1941       {
1942         if (is_del)
1943           {
1944             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1945               {
1946                 if (is_inside)
1947                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1948                 else
1949                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1950
1951                 if (sm->num_workers > 1 && !sm->deterministic)
1952                   {
1953                     del_feature_name = "nat44-handoff-classify";
1954                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1955                                                  "nat44-out2in-worker-handoff";
1956                   }
1957                 else if (sm->deterministic)
1958                   {
1959                     del_feature_name = "nat44-det-classify";
1960                     feature_name = !is_inside ?  "nat44-det-in2out" :
1961                                                  "nat44-det-out2in";
1962                   }
1963                 else if (sm->endpoint_dependent)
1964                   {
1965                     del_feature_name = "nat44-ed-classify";
1966                     feature_name = !is_inside ?  "nat-pre-in2out" :
1967                                                  "nat-pre-out2in";
1968                   }
1969                 else
1970                   {
1971                     del_feature_name = "nat44-classify";
1972                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1973                   }
1974
1975                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1976                 if (rv)
1977                   return rv;
1978                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1979                                              sw_if_index, 0, 0, 0);
1980                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1981                                              sw_if_index, 1, 0, 0);
1982                 if (!is_inside)
1983                   {
1984                     if (sm->endpoint_dependent)
1985                       vnet_feature_enable_disable ("ip4-local",
1986                                                    "nat44-ed-hairpinning",
1987                                                    sw_if_index, 1, 0, 0);
1988                     else if (!sm->deterministic)
1989                       vnet_feature_enable_disable ("ip4-local",
1990                                                    "nat44-hairpinning",
1991                                                    sw_if_index, 1, 0, 0);
1992                   }
1993               }
1994             else
1995               {
1996                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1997                 if (rv)
1998                   return rv;
1999                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2000                                              sw_if_index, 0, 0, 0);
2001                 pool_put (sm->interfaces, i);
2002                 if (is_inside)
2003                   {
2004                     if (sm->endpoint_dependent)
2005                       vnet_feature_enable_disable ("ip4-local",
2006                                                    "nat44-ed-hairpinning",
2007                                                    sw_if_index, 0, 0, 0);
2008                     else if (!sm->deterministic)
2009                       vnet_feature_enable_disable ("ip4-local",
2010                                                    "nat44-hairpinning",
2011                                                    sw_if_index, 0, 0, 0);
2012                   }
2013               }
2014           }
2015         else
2016           {
2017             if ((nat_interface_is_inside(i) && is_inside) ||
2018                 (nat_interface_is_outside(i) && !is_inside))
2019               return 0;
2020
2021             if (sm->num_workers > 1 && !sm->deterministic)
2022               {
2023                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2024                                                  "nat44-out2in-worker-handoff";
2025                 feature_name = "nat44-handoff-classify";
2026               }
2027             else if (sm->deterministic)
2028               {
2029                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
2030                                                  "nat44-det-out2in";
2031                 feature_name = "nat44-det-classify";
2032               }
2033             else if (sm->endpoint_dependent)
2034               {
2035                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2036                                                  "nat-pre-out2in";
2037
2038                 feature_name = "nat44-ed-classify";
2039               }
2040             else
2041               {
2042                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2043                 feature_name = "nat44-classify";
2044               }
2045
2046             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2047             if (rv)
2048               return rv;
2049             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2050                                          sw_if_index, 0, 0, 0);
2051             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2052                                          sw_if_index, 1, 0, 0);
2053             if (!is_inside)
2054               {
2055                 if (sm->endpoint_dependent)
2056                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2057                                                sw_if_index, 0, 0, 0);
2058                 else if (!sm->deterministic)
2059                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2060                                                sw_if_index, 0, 0, 0);
2061               }
2062             goto set_flags;
2063           }
2064
2065         goto fib;
2066       }
2067   }));
2068   /* *INDENT-ON* */
2069
2070   if (is_del)
2071     return VNET_API_ERROR_NO_SUCH_ENTRY;
2072
2073   pool_get (sm->interfaces, i);
2074   i->sw_if_index = sw_if_index;
2075   i->flags = 0;
2076   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2077                                0);
2078
2079   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2080   if (rv)
2081     return rv;
2082
2083   if (is_inside && !sm->out2in_dpo)
2084     {
2085       if (sm->endpoint_dependent)
2086         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2087                                      sw_if_index, 1, 0, 0);
2088       else if (!sm->deterministic)
2089         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2090                                      sw_if_index, 1, 0, 0);
2091     }
2092
2093 set_flags:
2094   if (is_inside)
2095     {
2096       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2097       return 0;
2098     }
2099   else
2100     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2101
2102   /* Add/delete external addresses to FIB */
2103 fib:
2104   /* *INDENT-OFF* */
2105   vec_foreach (ap, sm->addresses)
2106     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2107
2108   pool_foreach (m, sm->static_mappings,
2109   ({
2110     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2111       continue;
2112
2113     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2114   }));
2115
2116   pool_foreach (dm, sm->det_maps,
2117   ({
2118     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2119   }));
2120   /* *INDENT-ON* */
2121
2122   return 0;
2123 }
2124
2125 int
2126 snat_interface_add_del_output_feature (u32 sw_if_index,
2127                                        u8 is_inside, int is_del)
2128 {
2129   snat_main_t *sm = &snat_main;
2130   snat_interface_t *i;
2131   snat_address_t *ap;
2132   snat_static_mapping_t *m;
2133   nat_outside_fib_t *outside_fib;
2134   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2135                                                        sw_if_index);
2136
2137
2138   if (sm->deterministic ||
2139       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2140     return VNET_API_ERROR_UNSUPPORTED;
2141
2142   /* *INDENT-OFF* */
2143   pool_foreach (i, sm->interfaces,
2144   ({
2145     if (i->sw_if_index == sw_if_index)
2146       return VNET_API_ERROR_VALUE_EXIST;
2147   }));
2148   /* *INDENT-ON* */
2149
2150   if (!is_inside)
2151     {
2152       /* *INDENT-OFF* */
2153       vec_foreach (outside_fib, sm->outside_fibs)
2154         {
2155           if (outside_fib->fib_index == fib_index)
2156             {
2157               if (is_del)
2158                 {
2159                   outside_fib->refcount--;
2160                   if (!outside_fib->refcount)
2161                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2162                 }
2163               else
2164                 outside_fib->refcount++;
2165               goto feature_set;
2166             }
2167         }
2168       /* *INDENT-ON* */
2169       if (!is_del)
2170         {
2171           vec_add2 (sm->outside_fibs, outside_fib, 1);
2172           outside_fib->refcount = 1;
2173           outside_fib->fib_index = fib_index;
2174         }
2175     }
2176
2177 feature_set:
2178   if (is_inside)
2179     {
2180       if (sm->endpoint_dependent)
2181         {
2182           int rv =
2183             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2184           if (rv)
2185             return rv;
2186           rv =
2187             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2188                                                             !is_del);
2189           if (rv)
2190             return rv;
2191           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2192                                        sw_if_index, !is_del, 0, 0);
2193           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2194                                        sw_if_index, !is_del, 0, 0);
2195         }
2196       else
2197         {
2198           int rv =
2199             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2200           if (rv)
2201             return rv;
2202           rv =
2203             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2204                                                             !is_del);
2205           if (rv)
2206             return rv;
2207           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2208                                        sw_if_index, !is_del, 0, 0);
2209           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2210                                        sw_if_index, !is_del, 0, 0);
2211         }
2212       goto fq;
2213     }
2214
2215   if (sm->num_workers > 1)
2216     {
2217       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2218       if (rv)
2219         return rv;
2220       rv =
2221         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2222       if (rv)
2223         return rv;
2224       vnet_feature_enable_disable ("ip4-unicast",
2225                                    "nat44-out2in-worker-handoff",
2226                                    sw_if_index, !is_del, 0, 0);
2227       vnet_feature_enable_disable ("ip4-output",
2228                                    "nat44-in2out-output-worker-handoff",
2229                                    sw_if_index, !is_del, 0, 0);
2230     }
2231   else
2232     {
2233       if (sm->endpoint_dependent)
2234         {
2235           int rv =
2236             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2237           if (rv)
2238             return rv;
2239           rv =
2240             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2241                                                             !is_del);
2242           if (rv)
2243             return rv;
2244           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2245                                        sw_if_index, !is_del, 0, 0);
2246           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2247                                        sw_if_index, !is_del, 0, 0);
2248         }
2249       else
2250         {
2251           int rv =
2252             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2253           if (rv)
2254             return rv;
2255           rv =
2256             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2257                                                             !is_del);
2258           if (rv)
2259             return rv;
2260           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2261                                        sw_if_index, !is_del, 0, 0);
2262           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2263                                        sw_if_index, !is_del, 0, 0);
2264         }
2265     }
2266
2267 fq:
2268   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2269     sm->fq_in2out_output_index =
2270       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2271
2272   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2273     sm->fq_out2in_index =
2274       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2275
2276   /* *INDENT-OFF* */
2277   pool_foreach (i, sm->output_feature_interfaces,
2278   ({
2279     if (i->sw_if_index == sw_if_index)
2280       {
2281         if (is_del)
2282           pool_put (sm->output_feature_interfaces, i);
2283         else
2284           return VNET_API_ERROR_VALUE_EXIST;
2285
2286         goto fib;
2287       }
2288   }));
2289   /* *INDENT-ON* */
2290
2291   if (is_del)
2292     return VNET_API_ERROR_NO_SUCH_ENTRY;
2293
2294   pool_get (sm->output_feature_interfaces, i);
2295   i->sw_if_index = sw_if_index;
2296   i->flags = 0;
2297   if (is_inside)
2298     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2299   else
2300     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2301
2302   /* Add/delete external addresses to FIB */
2303 fib:
2304   if (is_inside)
2305     return 0;
2306
2307   /* *INDENT-OFF* */
2308   vec_foreach (ap, sm->addresses)
2309     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2310
2311   pool_foreach (m, sm->static_mappings,
2312   ({
2313     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2314       continue;
2315
2316     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2317   }));
2318   /* *INDENT-ON* */
2319
2320   return 0;
2321 }
2322
2323 int
2324 snat_set_workers (uword * bitmap)
2325 {
2326   snat_main_t *sm = &snat_main;
2327   int i, j = 0;
2328
2329   if (sm->num_workers < 2)
2330     return VNET_API_ERROR_FEATURE_DISABLED;
2331
2332   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2333     return VNET_API_ERROR_INVALID_WORKER;
2334
2335   vec_free (sm->workers);
2336   /* *INDENT-OFF* */
2337   clib_bitmap_foreach (i, bitmap,
2338     ({
2339       vec_add1(sm->workers, i);
2340       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2341       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2342       j++;
2343     }));
2344   /* *INDENT-ON* */
2345
2346   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2347   sm->num_snat_thread = _vec_len (sm->workers);
2348
2349   return 0;
2350 }
2351
2352 static void
2353 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2354                          u32 old_fib_index)
2355 {
2356   snat_main_t *sm = &snat_main;
2357   nat_outside_fib_t *outside_fib;
2358   snat_interface_t *i;
2359   u8 is_add = 1;
2360   u8 match = 0;
2361
2362   if (new_fib_index == old_fib_index)
2363     return;
2364
2365   if (!vec_len (sm->outside_fibs))
2366     return;
2367
2368   /* *INDENT-OFF* */
2369   pool_foreach (i, sm->interfaces,
2370     ({
2371       if (i->sw_if_index == sw_if_index)
2372         {
2373           if (!(nat_interface_is_outside (i)))
2374             return;
2375           match = 1;
2376         }
2377     }));
2378
2379   pool_foreach (i, sm->output_feature_interfaces,
2380     ({
2381       if (i->sw_if_index == sw_if_index)
2382         {
2383           if (!(nat_interface_is_outside (i)))
2384             return;
2385           match = 1;
2386         }
2387     }));
2388   /* *INDENT-ON* */
2389
2390   if (!match)
2391     return;
2392
2393   vec_foreach (outside_fib, sm->outside_fibs)
2394   {
2395     if (outside_fib->fib_index == old_fib_index)
2396       {
2397         outside_fib->refcount--;
2398         if (!outside_fib->refcount)
2399           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2400         break;
2401       }
2402   }
2403
2404   vec_foreach (outside_fib, sm->outside_fibs)
2405   {
2406     if (outside_fib->fib_index == new_fib_index)
2407       {
2408         outside_fib->refcount++;
2409         is_add = 0;
2410         break;
2411       }
2412   }
2413
2414   if (is_add)
2415     {
2416       vec_add2 (sm->outside_fibs, outside_fib, 1);
2417       outside_fib->refcount = 1;
2418       outside_fib->fib_index = new_fib_index;
2419     }
2420 }
2421
2422 static void
2423 snat_ip4_table_bind (ip4_main_t * im,
2424                      uword opaque,
2425                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2426 {
2427   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2428 }
2429
2430 static void
2431 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2432                                        uword opaque,
2433                                        u32 sw_if_index,
2434                                        ip4_address_t * address,
2435                                        u32 address_length,
2436                                        u32 if_address_index, u32 is_delete);
2437
2438 static void
2439 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2440                                  uword opaque,
2441                                  u32 sw_if_index,
2442                                  ip4_address_t * address,
2443                                  u32 address_length,
2444                                  u32 if_address_index, u32 is_delete);
2445
2446 static int
2447 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2448                                  u32 fib_index,
2449                                  u32 thread_index,
2450                                  snat_session_key_t * k,
2451                                  u16 port_per_thread, u32 snat_thread_index);
2452
2453 void
2454 test_ed_make_split ()
2455 {
2456   ip4_address_t l_addr;
2457   l_addr.as_u8[0] = 1;
2458   l_addr.as_u8[1] = 1;
2459   l_addr.as_u8[2] = 1;
2460   l_addr.as_u8[3] = 1;
2461   ip4_address_t r_addr;
2462   r_addr.as_u8[0] = 2;
2463   r_addr.as_u8[1] = 2;
2464   r_addr.as_u8[2] = 2;
2465   r_addr.as_u8[3] = 2;
2466   u16 l_port = 40001;
2467   u16 r_port = 40301;
2468   u8 proto = 9;
2469   u32 fib_index = 9000001;
2470   u64 value = ~0ULL;
2471   clib_bihash_kv_16_8_t kv;
2472   make_ed_kv (&l_addr, &r_addr, proto, fib_index, l_port, r_port, value, &kv);
2473   ip4_address_t l_addr2;
2474   ip4_address_t r_addr2;
2475   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2476   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2477   u16 l_port2 = 0;
2478   u16 r_port2 = 0;
2479   u8 proto2 = 0;
2480   u32 fib_index2 = 0;
2481   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2482                &r_port2);
2483   u64 value2 = kv.value;
2484   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2485   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2486   ASSERT (l_port == l_port2);
2487   ASSERT (r_port == r_port2);
2488   ASSERT (proto == proto2);
2489   ASSERT (fib_index == fib_index2);
2490   ASSERT (value == value2);
2491 }
2492
2493 static clib_error_t *
2494 snat_init (vlib_main_t * vm)
2495 {
2496   snat_main_t *sm = &snat_main;
2497   clib_error_t *error = 0;
2498   ip4_main_t *im = &ip4_main;
2499   ip_lookup_main_t *lm = &im->lookup_main;
2500   uword *p;
2501   vlib_thread_registration_t *tr;
2502   vlib_thread_main_t *tm = vlib_get_thread_main ();
2503   uword *bitmap = 0;
2504   u32 i;
2505   ip4_add_del_interface_address_callback_t cb4;
2506   vlib_node_t *node;
2507
2508   sm->vnet_main = vnet_get_main ();
2509   sm->ip4_main = im;
2510   sm->ip4_lookup_main = lm;
2511   sm->api_main = vlibapi_get_main ();
2512   sm->first_worker_index = 0;
2513   sm->num_workers = 0;
2514   sm->num_snat_thread = 1;
2515   sm->workers = 0;
2516   sm->port_per_thread = 0xffff - 1024;
2517   sm->fq_in2out_index = ~0;
2518   sm->fq_in2out_output_index = ~0;
2519   sm->fq_out2in_index = ~0;
2520
2521   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2522   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2523   sm->forwarding_enabled = 0;
2524   sm->log_class = vlib_log_register_class ("nat", 0);
2525   sm->log_level = SNAT_LOG_ERROR;
2526   sm->mss_clamping = 0;
2527
2528   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2529   sm->error_node_index = node->index;
2530
2531   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2532   sm->pre_in2out_node_index = node->index;
2533   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2534   sm->pre_out2in_node_index = node->index;
2535
2536   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2537   sm->pre_in2out_node_index = node->index;
2538
2539   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2540   sm->pre_out2in_node_index = node->index;
2541
2542   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2543   sm->in2out_node_index = node->index;
2544   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2545   sm->in2out_output_node_index = node->index;
2546   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2547   sm->in2out_fast_node_index = node->index;
2548   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2549   sm->in2out_slowpath_node_index = node->index;
2550   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2551   sm->in2out_slowpath_output_node_index = node->index;
2552
2553   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2554   sm->ed_in2out_node_index = node->index;
2555   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2556   sm->ed_in2out_slowpath_node_index = node->index;
2557
2558   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2559   sm->out2in_node_index = node->index;
2560   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2561   sm->out2in_fast_node_index = node->index;
2562
2563   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2564   sm->ed_out2in_node_index = node->index;
2565   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2566   sm->ed_out2in_slowpath_node_index = node->index;
2567
2568   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2569   sm->det_in2out_node_index = node->index;
2570   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2571   sm->det_out2in_node_index = node->index;
2572
2573   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2574   sm->hairpinning_node_index = node->index;
2575   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2576   sm->hairpin_dst_node_index = node->index;
2577   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2578   sm->hairpin_src_node_index = node->index;
2579   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2580   sm->ed_hairpinning_node_index = node->index;
2581   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2582   sm->ed_hairpin_dst_node_index = node->index;
2583   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2584   sm->ed_hairpin_src_node_index = node->index;
2585
2586   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2587   if (p)
2588     {
2589       tr = (vlib_thread_registration_t *) p[0];
2590       if (tr)
2591         {
2592           sm->num_workers = tr->count;
2593           sm->first_worker_index = tr->first_index;
2594         }
2595     }
2596
2597   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2598
2599   /* Use all available workers by default */
2600   if (sm->num_workers > 1)
2601     {
2602       for (i = 0; i < sm->num_workers; i++)
2603         bitmap = clib_bitmap_set (bitmap, i, 1);
2604       // sets thread indexes for workes
2605       snat_set_workers (bitmap);
2606       clib_bitmap_free (bitmap);
2607     }
2608   else
2609     {
2610       sm->per_thread_data[0].snat_thread_index = 0;
2611     }
2612
2613   error = snat_api_init (vm, sm);
2614   if (error)
2615     return error;
2616
2617   /* Set up the interface address add/del callback */
2618   cb4.function = snat_ip4_add_del_interface_address_cb;
2619   cb4.function_opaque = 0;
2620
2621   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2622
2623   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2624   cb4.function_opaque = 0;
2625
2626   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2627
2628   nat_dpo_module_init ();
2629
2630   /* Init counters */
2631   sm->total_users.name = "total-users";
2632   sm->total_users.stat_segment_name = "/nat44/total-users";
2633   vlib_validate_simple_counter (&sm->total_users, 0);
2634   vlib_zero_simple_counter (&sm->total_users, 0);
2635   sm->total_sessions.name = "total-sessions";
2636   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2637   vlib_validate_simple_counter (&sm->total_sessions, 0);
2638   vlib_zero_simple_counter (&sm->total_sessions, 0);
2639
2640   /* Init IPFIX logging */
2641   snat_ipfix_logging_init (vm);
2642
2643   /* Init NAT64 */
2644   error = nat64_init (vm);
2645   if (error)
2646     return error;
2647
2648   nat66_init (vm);
2649
2650   ip4_table_bind_callback_t cbt4 = {
2651     .function = snat_ip4_table_bind,
2652   };
2653   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2654
2655   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2656                                         FIB_SOURCE_PRIORITY_HI,
2657                                         FIB_SOURCE_BH_SIMPLE);
2658   nat_fib_src_low = fib_source_allocate ("nat-low",
2659                                          FIB_SOURCE_PRIORITY_LOW,
2660                                          FIB_SOURCE_BH_SIMPLE);
2661
2662   test_ed_make_split ();
2663   return error;
2664 }
2665
2666 VLIB_INIT_FUNCTION (snat_init);
2667
2668 void
2669 snat_free_outside_address_and_port (snat_address_t * addresses,
2670                                     u32 thread_index, snat_session_key_t * k)
2671 {
2672   snat_address_t *a;
2673   u32 address_index;
2674   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2675
2676   for (address_index = 0; address_index < vec_len (addresses);
2677        address_index++)
2678     {
2679       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2680         break;
2681     }
2682
2683   ASSERT (address_index < vec_len (addresses));
2684
2685   a = addresses + address_index;
2686
2687   switch (k->protocol)
2688     {
2689 #define _(N, i, n, s) \
2690     case NAT_PROTOCOL_##N: \
2691       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2692       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2693       a->busy_##n##_ports--; \
2694       a->busy_##n##_ports_per_thread[thread_index]--; \
2695       break;
2696       foreach_nat_protocol
2697 #undef _
2698     default:
2699       nat_elog_info ("unknown protocol");
2700       return;
2701     }
2702 }
2703
2704 static int
2705 nat_set_outside_address_and_port (snat_address_t * addresses,
2706                                   u32 thread_index, snat_session_key_t * k)
2707 {
2708   snat_address_t *a = 0;
2709   u32 address_index;
2710   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2711
2712   for (address_index = 0; address_index < vec_len (addresses);
2713        address_index++)
2714     {
2715       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2716         continue;
2717
2718       a = addresses + address_index;
2719       switch (k->protocol)
2720         {
2721 #define _(N, j, n, s) \
2722         case NAT_PROTOCOL_##N: \
2723           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2724             return VNET_API_ERROR_INSTANCE_IN_USE; \
2725           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2726           a->busy_##n##_ports_per_thread[thread_index]++; \
2727           a->busy_##n##_ports++; \
2728           return 0;
2729           foreach_nat_protocol
2730 #undef _
2731         default:
2732           nat_elog_info ("unknown protocol");
2733           return 1;
2734         }
2735     }
2736
2737   return VNET_API_ERROR_NO_SUCH_ENTRY;
2738 }
2739
2740 int
2741 snat_static_mapping_match (snat_main_t * sm,
2742                            snat_session_key_t match,
2743                            snat_session_key_t * mapping,
2744                            u8 by_external,
2745                            u8 * is_addr_only,
2746                            twice_nat_type_t * twice_nat,
2747                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2748                            u8 * is_identity_nat)
2749 {
2750   clib_bihash_kv_8_8_t kv, value;
2751   snat_static_mapping_t *m;
2752   snat_session_key_t m_key;
2753   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2754   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2755   u8 backend_index;
2756   nat44_lb_addr_port_t *local;
2757
2758   m_key.fib_index = match.fib_index;
2759   if (by_external)
2760     {
2761       mapping_hash = &sm->static_mapping_by_external;
2762       m_key.fib_index = 0;
2763     }
2764
2765   m_key.addr = match.addr;
2766   m_key.port = clib_net_to_host_u16 (match.port);
2767   m_key.protocol = match.protocol;
2768
2769   kv.key = m_key.as_u64;
2770
2771   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2772     {
2773       /* Try address only mapping */
2774       m_key.port = 0;
2775       m_key.protocol = 0;
2776       kv.key = m_key.as_u64;
2777       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2778         return 1;
2779     }
2780
2781   m = pool_elt_at_index (sm->static_mappings, value.value);
2782
2783   if (by_external)
2784     {
2785       if (is_lb_static_mapping (m))
2786         {
2787           if (PREDICT_FALSE (lb != 0))
2788             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2789           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2790                                                           match.addr,
2791                                                           match.protocol,
2792                                                           match.port,
2793                                                           &backend_index))
2794             {
2795               local = pool_elt_at_index (m->locals, backend_index);
2796               mapping->addr = local->addr;
2797               mapping->port = clib_host_to_net_u16 (local->port);
2798               mapping->fib_index = local->fib_index;
2799               goto end;
2800             }
2801           // pick locals matching this worker
2802           if (PREDICT_FALSE (sm->num_workers > 1))
2803             {
2804               u32 thread_index = vlib_get_thread_index ();
2805               /* *INDENT-OFF* */
2806               pool_foreach_index (i, m->locals,
2807               ({
2808                 local = pool_elt_at_index (m->locals, i);
2809
2810                 ip4_header_t ip = {
2811                   .src_address = local->addr,
2812                 };
2813
2814                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2815                     thread_index)
2816                   {
2817                     vec_add1 (tmp, i);
2818                   }
2819               }));
2820               /* *INDENT-ON* */
2821               ASSERT (vec_len (tmp) != 0);
2822             }
2823           else
2824             {
2825               /* *INDENT-OFF* */
2826               pool_foreach_index (i, m->locals,
2827               ({
2828                 vec_add1 (tmp, i);
2829               }));
2830               /* *INDENT-ON* */
2831             }
2832           hi = vec_len (tmp) - 1;
2833           local = pool_elt_at_index (m->locals, tmp[hi]);
2834           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2835           while (lo < hi)
2836             {
2837               mid = ((hi - lo) >> 1) + lo;
2838               local = pool_elt_at_index (m->locals, tmp[mid]);
2839               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2840             }
2841           local = pool_elt_at_index (m->locals, tmp[lo]);
2842           if (!(local->prefix >= rand))
2843             return 1;
2844           mapping->addr = local->addr;
2845           mapping->port = clib_host_to_net_u16 (local->port);
2846           mapping->fib_index = local->fib_index;
2847           if (m->affinity)
2848             {
2849               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2850                                                 match.protocol, match.port,
2851                                                 tmp[lo], m->affinity,
2852                                                 m->affinity_per_service_list_head_index))
2853                 nat_elog_info ("create affinity record failed");
2854             }
2855           vec_free (tmp);
2856         }
2857       else
2858         {
2859           if (PREDICT_FALSE (lb != 0))
2860             *lb = NO_LB_NAT;
2861           mapping->fib_index = m->fib_index;
2862           mapping->addr = m->local_addr;
2863           /* Address only mapping doesn't change port */
2864           mapping->port = is_addr_only_static_mapping (m) ? match.port
2865             : clib_host_to_net_u16 (m->local_port);
2866         }
2867       mapping->protocol = m->proto;
2868     }
2869   else
2870     {
2871       mapping->addr = m->external_addr;
2872       /* Address only mapping doesn't change port */
2873       mapping->port = is_addr_only_static_mapping (m) ? match.port
2874         : clib_host_to_net_u16 (m->external_port);
2875       mapping->fib_index = sm->outside_fib_index;
2876     }
2877
2878 end:
2879   if (PREDICT_FALSE (is_addr_only != 0))
2880     *is_addr_only = is_addr_only_static_mapping (m);
2881
2882   if (PREDICT_FALSE (twice_nat != 0))
2883     *twice_nat = m->twice_nat;
2884
2885   if (PREDICT_FALSE (is_identity_nat != 0))
2886     *is_identity_nat = is_identity_static_mapping (m);
2887
2888   return 0;
2889 }
2890
2891 static_always_inline u16
2892 snat_random_port (u16 min, u16 max)
2893 {
2894   snat_main_t *sm = &snat_main;
2895   return min + random_u32 (&sm->random_seed) /
2896     (random_u32_max () / (max - min + 1) + 1);
2897 }
2898
2899 int
2900 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2901                                      u32 fib_index,
2902                                      u32 thread_index,
2903                                      snat_session_key_t * k,
2904                                      u16 port_per_thread,
2905                                      u32 snat_thread_index)
2906 {
2907   snat_main_t *sm = &snat_main;
2908
2909   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2910                                   port_per_thread, snat_thread_index);
2911 }
2912
2913 static int
2914 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2915                                  u32 fib_index,
2916                                  u32 thread_index,
2917                                  snat_session_key_t * k,
2918                                  u16 port_per_thread, u32 snat_thread_index)
2919 {
2920   int i;
2921   snat_address_t *a, *ga = 0;
2922   u32 portnum;
2923
2924   for (i = 0; i < vec_len (addresses); i++)
2925     {
2926       a = addresses + i;
2927       switch (k->protocol)
2928         {
2929 #define _(N, j, n, s) \
2930         case NAT_PROTOCOL_##N: \
2931           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2932             { \
2933               if (a->fib_index == fib_index) \
2934                 { \
2935                   while (1) \
2936                     { \
2937                       portnum = (port_per_thread * \
2938                         snat_thread_index) + \
2939                         snat_random_port(1, port_per_thread) + 1024; \
2940                       if (a->busy_##n##_port_refcounts[portnum]) \
2941                         continue; \
2942                       --a->busy_##n##_port_refcounts[portnum]; \
2943                       a->busy_##n##_ports_per_thread[thread_index]++; \
2944                       a->busy_##n##_ports++; \
2945                       k->addr = a->addr; \
2946                       k->port = clib_host_to_net_u16(portnum); \
2947                       return 0; \
2948                     } \
2949                 } \
2950               else if (a->fib_index == ~0) \
2951                 { \
2952                   ga = a; \
2953                 } \
2954             } \
2955           break;
2956           foreach_nat_protocol
2957 #undef _
2958         default:
2959           nat_elog_info ("unknown protocol");
2960           return 1;
2961         }
2962
2963     }
2964
2965   if (ga)
2966     {
2967       a = ga;
2968       switch (k->protocol)
2969         {
2970 #define _(N, j, n, s) \
2971         case NAT_PROTOCOL_##N: \
2972           while (1) \
2973             { \
2974               portnum = (port_per_thread * \
2975                 snat_thread_index) + \
2976                 snat_random_port(1, port_per_thread) + 1024; \
2977               if (a->busy_##n##_port_refcounts[portnum]) \
2978                 continue; \
2979               ++a->busy_##n##_port_refcounts[portnum]; \
2980               a->busy_##n##_ports_per_thread[thread_index]++; \
2981               a->busy_##n##_ports++; \
2982               k->addr = a->addr; \
2983               k->port = clib_host_to_net_u16(portnum); \
2984               return 0; \
2985             }
2986           break;
2987           foreach_nat_protocol
2988 #undef _
2989         default:
2990           nat_elog_info ("unknown protocol");
2991           return 1;
2992         }
2993     }
2994
2995   /* Totally out of translations to use... */
2996   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2997   return 1;
2998 }
2999
3000 static int
3001 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
3002                               u32 fib_index,
3003                               u32 thread_index,
3004                               snat_session_key_t * k,
3005                               u16 port_per_thread, u32 snat_thread_index)
3006 {
3007   snat_main_t *sm = &snat_main;
3008   snat_address_t *a = addresses;
3009   u16 m, ports, portnum, A, j;
3010   m = 16 - (sm->psid_offset + sm->psid_length);
3011   ports = (1 << (16 - sm->psid_length)) - (1 << m);
3012
3013   if (!vec_len (addresses))
3014     goto exhausted;
3015
3016   switch (k->protocol)
3017     {
3018 #define _(N, i, n, s) \
3019     case NAT_PROTOCOL_##N: \
3020       if (a->busy_##n##_ports < ports) \
3021         { \
3022           while (1) \
3023             { \
3024               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3025               j = snat_random_port(0, pow2_mask(m)); \
3026               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3027               if (a->busy_##n##_port_refcounts[portnum]) \
3028                 continue; \
3029               ++a->busy_##n##_port_refcounts[portnum]; \
3030               a->busy_##n##_ports++; \
3031               k->addr = a->addr; \
3032               k->port = clib_host_to_net_u16 (portnum); \
3033               return 0; \
3034             } \
3035         } \
3036       break;
3037       foreach_nat_protocol
3038 #undef _
3039     default:
3040       nat_elog_info ("unknown protocol");
3041       return 1;
3042     }
3043
3044 exhausted:
3045   /* Totally out of translations to use... */
3046   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3047   return 1;
3048 }
3049
3050 static int
3051 nat_alloc_addr_and_port_range (snat_address_t * addresses,
3052                                u32 fib_index,
3053                                u32 thread_index,
3054                                snat_session_key_t * k,
3055                                u16 port_per_thread, u32 snat_thread_index)
3056 {
3057   snat_main_t *sm = &snat_main;
3058   snat_address_t *a = addresses;
3059   u16 portnum, ports;
3060
3061   ports = sm->end_port - sm->start_port + 1;
3062
3063   if (!vec_len (addresses))
3064     goto exhausted;
3065
3066   switch (k->protocol)
3067     {
3068 #define _(N, i, n, s) \
3069     case NAT_PROTOCOL_##N: \
3070       if (a->busy_##n##_ports < ports) \
3071         { \
3072           while (1) \
3073             { \
3074               portnum = snat_random_port(sm->start_port, sm->end_port); \
3075               if (a->busy_##n##_port_refcounts[portnum]) \
3076                 continue; \
3077               ++a->busy_##n##_port_refcounts[portnum]; \
3078               a->busy_##n##_ports++; \
3079               k->addr = a->addr; \
3080               k->port = clib_host_to_net_u16 (portnum); \
3081               return 0; \
3082             } \
3083         } \
3084       break;
3085       foreach_nat_protocol
3086 #undef _
3087     default:
3088       nat_elog_info ("unknown protocol");
3089       return 1;
3090     }
3091
3092 exhausted:
3093   /* Totally out of translations to use... */
3094   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3095   return 1;
3096 }
3097
3098 void
3099 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3100 {
3101   dpo_id_t dpo_v4 = DPO_INVALID;
3102   fib_prefix_t pfx = {
3103     .fp_proto = FIB_PROTOCOL_IP4,
3104     .fp_len = 32,
3105     .fp_addr.ip4.as_u32 = addr.as_u32,
3106   };
3107
3108   if (is_add)
3109     {
3110       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3111       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3112                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3113       dpo_reset (&dpo_v4);
3114     }
3115   else
3116     {
3117       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3118     }
3119 }
3120
3121 u8 *
3122 format_session_kvp (u8 * s, va_list * args)
3123 {
3124   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3125   snat_session_key_t k;
3126
3127   k.as_u64 = v->key;
3128
3129   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
3130
3131   return s;
3132 }
3133
3134 u8 *
3135 format_static_mapping_kvp (u8 * s, va_list * args)
3136 {
3137   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3138   snat_session_key_t k;
3139
3140   k.as_u64 = v->key;
3141
3142   s = format (s, "%U static-mapping-index %llu",
3143               format_static_mapping_key, &k, v->value);
3144
3145   return s;
3146 }
3147
3148 u8 *
3149 format_user_kvp (u8 * s, va_list * args)
3150 {
3151   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3152   snat_user_key_t k;
3153
3154   k.as_u64 = v->key;
3155
3156   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3157               k.fib_index, v->value);
3158
3159   return s;
3160 }
3161
3162 u8 *
3163 format_ed_session_kvp (u8 * s, va_list * args)
3164 {
3165   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3166
3167   u8 proto;
3168   u16 r_port, l_port;
3169   ip4_address_t l_addr, r_addr;
3170   u32 fib_index;
3171
3172   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
3173   s =
3174     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3175             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
3176             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
3177             format_ip_protocol, proto, fib_index, v->value);
3178
3179   return s;
3180 }
3181
3182 static u32
3183 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3184                            u8 is_output)
3185 {
3186   snat_main_t *sm = &snat_main;
3187   u32 next_worker_index = 0;
3188   u32 hash;
3189
3190   next_worker_index = sm->first_worker_index;
3191   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3192     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3193
3194   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3195     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3196   else
3197     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3198
3199   return next_worker_index;
3200 }
3201
3202 static u32
3203 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3204                            u32 rx_fib_index0, u8 is_output)
3205 {
3206   snat_main_t *sm = &snat_main;
3207   udp_header_t *udp;
3208   u16 port;
3209   snat_session_key_t m_key;
3210   clib_bihash_kv_8_8_t kv, value;
3211   snat_static_mapping_t *m;
3212   u32 proto;
3213   u32 next_worker_index = 0;
3214
3215   /* first try static mappings without port */
3216   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3217     {
3218       m_key.addr = ip0->dst_address;
3219       m_key.port = 0;
3220       m_key.protocol = 0;
3221       m_key.fib_index = rx_fib_index0;
3222       kv.key = m_key.as_u64;
3223       if (!clib_bihash_search_8_8
3224           (&sm->static_mapping_by_external, &kv, &value))
3225         {
3226           m = pool_elt_at_index (sm->static_mappings, value.value);
3227           return m->workers[0];
3228         }
3229     }
3230
3231   proto = ip_proto_to_nat_proto (ip0->protocol);
3232   udp = ip4_next_header (ip0);
3233   port = udp->dst_port;
3234
3235   /* unknown protocol */
3236   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3237     {
3238       /* use current thread */
3239       return vlib_get_thread_index ();
3240     }
3241
3242   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3243     {
3244       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3245       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3246       if (!icmp_type_is_error_message
3247           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3248         port = vnet_buffer (b)->ip.reass.l4_src_port;
3249       else
3250         {
3251           /* if error message, then it's not fragmented and we can access it */
3252           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3253           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3254           void *l4_header = ip4_next_header (inner_ip);
3255           switch (proto)
3256             {
3257             case NAT_PROTOCOL_ICMP:
3258               icmp = (icmp46_header_t *) l4_header;
3259               echo = (icmp_echo_header_t *) (icmp + 1);
3260               port = echo->identifier;
3261               break;
3262             case NAT_PROTOCOL_UDP:
3263             case NAT_PROTOCOL_TCP:
3264               port = ((tcp_udp_header_t *) l4_header)->src_port;
3265               break;
3266             default:
3267               return vlib_get_thread_index ();
3268             }
3269         }
3270     }
3271
3272   /* try static mappings with port */
3273   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3274     {
3275       m_key.addr = ip0->dst_address;
3276       m_key.port = clib_net_to_host_u16 (port);
3277       m_key.protocol = proto;
3278       m_key.fib_index = rx_fib_index0;
3279       kv.key = m_key.as_u64;
3280       if (!clib_bihash_search_8_8
3281           (&sm->static_mapping_by_external, &kv, &value))
3282         {
3283           m = pool_elt_at_index (sm->static_mappings, value.value);
3284           return m->workers[0];
3285         }
3286     }
3287
3288   /* worker by outside port */
3289   next_worker_index = sm->first_worker_index;
3290   next_worker_index +=
3291     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3292   return next_worker_index;
3293 }
3294
3295 static u32
3296 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3297                                u8 is_output)
3298 {
3299   snat_main_t *sm = &snat_main;
3300   u32 next_worker_index = sm->first_worker_index;
3301   u32 hash;
3302
3303   clib_bihash_kv_16_8_t kv16, value16;
3304   snat_main_per_thread_data_t *tsm;
3305   udp_header_t *udp;
3306
3307   if (PREDICT_FALSE (is_output))
3308     {
3309       u32 fib_index = sm->outside_fib_index;
3310       nat_outside_fib_t *outside_fib;
3311       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3312       fib_prefix_t pfx = {
3313         .fp_proto = FIB_PROTOCOL_IP4,
3314         .fp_len = 32,
3315         .fp_addr = {
3316                     .ip4.as_u32 = ip->dst_address.as_u32,
3317                     }
3318         ,
3319       };
3320
3321       udp = ip4_next_header (ip);
3322
3323       switch (vec_len (sm->outside_fibs))
3324         {
3325         case 0:
3326           fib_index = sm->outside_fib_index;
3327           break;
3328         case 1:
3329           fib_index = sm->outside_fibs[0].fib_index;
3330           break;
3331         default:
3332             /* *INDENT-OFF* */
3333             vec_foreach (outside_fib, sm->outside_fibs)
3334               {
3335                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3336                 if (FIB_NODE_INDEX_INVALID != fei)
3337                   {
3338                     if (fib_entry_get_resolving_interface (fei) != ~0)
3339                       {
3340                         fib_index = outside_fib->fib_index;
3341                         break;
3342                       }
3343                   }
3344               }
3345             /* *INDENT-ON* */
3346           break;
3347         }
3348
3349       make_ed_kv (&ip->src_address, &ip->dst_address,
3350                   ip->protocol, fib_index, udp->src_port, udp->dst_port,
3351                   ~0ULL, &kv16);
3352
3353       /* *INDENT-OFF* */
3354       vec_foreach (tsm, sm->per_thread_data)
3355         {
3356           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3357                                                       &kv16, &value16)))
3358             {
3359               next_worker_index += tsm->thread_index;
3360
3361               nat_elog_debug_handoff (
3362                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3363                 next_worker_index, fib_index,
3364                 clib_net_to_host_u32 (ip->src_address.as_u32),
3365                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3366
3367               return next_worker_index;
3368             }
3369         }
3370       /* *INDENT-ON* */
3371     }
3372
3373   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3374     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3375
3376   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3377     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3378   else
3379     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3380
3381   if (PREDICT_TRUE (!is_output))
3382     {
3383       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3384                               next_worker_index, rx_fib_index,
3385                               clib_net_to_host_u32 (ip->src_address.as_u32),
3386                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3387     }
3388   else
3389     {
3390       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3391                               next_worker_index, rx_fib_index,
3392                               clib_net_to_host_u32 (ip->src_address.as_u32),
3393                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3394     }
3395
3396   return next_worker_index;
3397 }
3398
3399 static u32
3400 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3401                                u32 rx_fib_index, u8 is_output)
3402 {
3403   snat_main_t *sm = &snat_main;
3404   clib_bihash_kv_8_8_t kv, value;
3405   clib_bihash_kv_16_8_t kv16, value16;
3406   snat_main_per_thread_data_t *tsm;
3407
3408   u32 proto, next_worker_index = 0;
3409   udp_header_t *udp;
3410   u16 port;
3411   snat_static_mapping_t *m;
3412   u32 hash;
3413
3414   proto = ip_proto_to_nat_proto (ip->protocol);
3415
3416   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3417     {
3418       udp = ip4_next_header (ip);
3419
3420       make_ed_kv (&ip->dst_address, &ip->src_address,
3421                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port,
3422                   ~0ULL, &kv16);
3423
3424       /* *INDENT-OFF* */
3425       vec_foreach (tsm, sm->per_thread_data)
3426         {
3427           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3428                                                       &kv16, &value16)))
3429             {
3430               next_worker_index = sm->first_worker_index + tsm->thread_index;
3431               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3432                           next_worker_index, rx_fib_index,
3433                           clib_net_to_host_u32 (ip->src_address.as_u32),
3434                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3435               return next_worker_index;
3436             }
3437           }
3438         /* *INDENT-ON* */
3439     }
3440   else if (proto == NAT_PROTOCOL_ICMP)
3441     {
3442       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0ULL, 0, 0, 0, &kv16))
3443         {
3444           /* *INDENT-OFF* */
3445           vec_foreach (tsm, sm->per_thread_data)
3446             {
3447               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3448                                                           &kv16, &value16)))
3449                 {
3450                   next_worker_index = sm->first_worker_index +
3451                                       tsm->thread_index;
3452                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3453                               next_worker_index, rx_fib_index,
3454                               clib_net_to_host_u32 (ip->src_address.as_u32),
3455                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3456                   return next_worker_index;
3457                 }
3458             }
3459           /* *INDENT-ON* */
3460         }
3461     }
3462
3463   /* first try static mappings without port */
3464   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3465     {
3466       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3467       if (!clib_bihash_search_8_8
3468           (&sm->static_mapping_by_external, &kv, &value))
3469         {
3470           m = pool_elt_at_index (sm->static_mappings, value.value);
3471           next_worker_index = m->workers[0];
3472           goto done;
3473         }
3474     }
3475
3476   /* unknown protocol */
3477   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3478     {
3479       /* use current thread */
3480       next_worker_index = vlib_get_thread_index ();
3481       goto done;
3482     }
3483
3484   udp = ip4_next_header (ip);
3485   port = udp->dst_port;
3486
3487   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3488     {
3489       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3490       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3491       if (!icmp_type_is_error_message
3492           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3493         port = vnet_buffer (b)->ip.reass.l4_src_port;
3494       else
3495         {
3496           /* if error message, then it's not fragmented and we can access it */
3497           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3498           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3499           void *l4_header = ip4_next_header (inner_ip);
3500           switch (proto)
3501             {
3502             case NAT_PROTOCOL_ICMP:
3503               icmp = (icmp46_header_t *) l4_header;
3504               echo = (icmp_echo_header_t *) (icmp + 1);
3505               port = echo->identifier;
3506               break;
3507             case NAT_PROTOCOL_UDP:
3508             case NAT_PROTOCOL_TCP:
3509               port = ((tcp_udp_header_t *) l4_header)->src_port;
3510               break;
3511             default:
3512               next_worker_index = vlib_get_thread_index ();
3513               goto done;
3514             }
3515         }
3516     }
3517
3518   /* try static mappings with port */
3519   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3520     {
3521       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3522                   clib_net_to_host_u16 (port));
3523       if (!clib_bihash_search_8_8
3524           (&sm->static_mapping_by_external, &kv, &value))
3525         {
3526           m = pool_elt_at_index (sm->static_mappings, value.value);
3527           if (!is_lb_static_mapping (m))
3528             {
3529               next_worker_index = m->workers[0];
3530               goto done;
3531             }
3532
3533           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3534             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3535
3536           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3537             next_worker_index =
3538               m->workers[hash & (_vec_len (m->workers) - 1)];
3539           else
3540             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3541           goto done;
3542         }
3543     }
3544
3545   /* worker by outside port */
3546   next_worker_index = sm->first_worker_index;
3547   next_worker_index +=
3548     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3549
3550 done:
3551   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3552                           clib_net_to_host_u32 (ip->src_address.as_u32),
3553                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3554   return next_worker_index;
3555 }
3556
3557 void
3558 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3559                 ip4_address_t * out_addr, u16 out_port,
3560                 ip4_address_t * eh_addr, u16 eh_port,
3561                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3562                 u32 fib_index, u16 flags, u32 thread_index)
3563 {
3564   snat_main_t *sm = &snat_main;
3565   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3566   snat_session_key_t key;
3567   snat_user_t *u;
3568   snat_session_t *s;
3569   clib_bihash_kv_8_8_t kv;
3570   vlib_main_t *vm = vlib_get_main ();
3571   f64 now = vlib_time_now (vm);
3572   nat_outside_fib_t *outside_fib;
3573   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3574   fib_prefix_t pfx = {
3575     .fp_proto = FIB_PROTOCOL_IP4,
3576     .fp_len = 32,
3577     .fp_addr = {
3578                 .ip4.as_u32 = eh_addr->as_u32,
3579                 },
3580   };
3581
3582   key.addr.as_u32 = out_addr->as_u32;
3583   key.port = out_port;
3584   key.protocol = proto;
3585
3586   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3587     {
3588       if (nat_set_outside_address_and_port
3589           (sm->addresses, thread_index, &key))
3590         return;
3591     }
3592
3593   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3594   if (!u)
3595     return;
3596
3597   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3598   if (!s)
3599     return;
3600
3601   if (sm->endpoint_dependent)
3602     {
3603       nat_ed_lru_insert (tsm, s, now, proto);
3604     }
3605
3606   s->last_heard = now;
3607   s->flags = flags;
3608   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3609   s->ext_host_port = eh_port;
3610   user_session_increment (sm, u, snat_is_session_static (s));
3611   switch (vec_len (sm->outside_fibs))
3612     {
3613     case 0:
3614       key.fib_index = sm->outside_fib_index;
3615       break;
3616     case 1:
3617       key.fib_index = sm->outside_fibs[0].fib_index;
3618       break;
3619     default:
3620       /* *INDENT-OFF* */
3621       vec_foreach (outside_fib, sm->outside_fibs)
3622         {
3623           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3624           if (FIB_NODE_INDEX_INVALID != fei)
3625             {
3626               if (fib_entry_get_resolving_interface (fei) != ~0)
3627                 {
3628                   key.fib_index = outside_fib->fib_index;
3629                   break;
3630                 }
3631             }
3632         }
3633       /* *INDENT-ON* */
3634       break;
3635     }
3636   s->out2in = key;
3637   kv.key = key.as_u64;
3638   kv.value = s - tsm->sessions;
3639   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3640     nat_elog_warn ("out2in key add failed");
3641
3642   key.addr.as_u32 = in_addr->as_u32;
3643   key.port = in_port;
3644   key.fib_index = fib_index;
3645   s->in2out = key;
3646   kv.key = key.as_u64;
3647   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3648     nat_elog_warn ("in2out key add failed");
3649 }
3650
3651 void
3652 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3653                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3654                 u32 ti)
3655 {
3656   snat_main_t *sm = &snat_main;
3657   snat_session_key_t key;
3658   clib_bihash_kv_8_8_t kv, value;
3659   u32 thread_index;
3660   snat_session_t *s;
3661   snat_main_per_thread_data_t *tsm;
3662
3663   if (sm->num_workers > 1)
3664     thread_index =
3665       sm->first_worker_index +
3666       (sm->workers[(clib_net_to_host_u16 (out_port) -
3667                     1024) / sm->port_per_thread]);
3668   else
3669     thread_index = sm->num_workers;
3670   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3671
3672   key.addr.as_u32 = out_addr->as_u32;
3673   key.port = out_port;
3674   key.protocol = proto;
3675   key.fib_index = fib_index;
3676   kv.key = key.as_u64;
3677   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3678     return;
3679
3680   s = pool_elt_at_index (tsm->sessions, value.value);
3681   nat_free_session_data (sm, s, thread_index, 1);
3682   nat44_delete_session (sm, s, thread_index);
3683 }
3684
3685 void
3686 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3687                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3688                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3689 {
3690   snat_main_t *sm = &snat_main;
3691   snat_session_key_t key;
3692   clib_bihash_kv_8_8_t kv, value;
3693   snat_session_t *s;
3694   snat_main_per_thread_data_t *tsm;
3695
3696   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3697
3698   key.addr.as_u32 = out_addr->as_u32;
3699   key.port = out_port;
3700   key.protocol = proto;
3701   key.fib_index = fib_index;
3702   kv.key = key.as_u64;
3703   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3704     return;
3705
3706   s = pool_elt_at_index (tsm->sessions, value.value);
3707   s->total_pkts = total_pkts;
3708   s->total_bytes = total_bytes;
3709 }
3710
3711 void
3712 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3713                    ip4_address_t * out_addr, u16 out_port,
3714                    ip4_address_t * eh_addr, u16 eh_port,
3715                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3716                    u32 fib_index, u16 flags, u32 thread_index)
3717 {
3718   snat_main_t *sm = &snat_main;
3719   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3720   snat_session_key_t key;
3721   snat_session_t *s;
3722   clib_bihash_kv_16_8_t kv;
3723   vlib_main_t *vm = vlib_get_main ();
3724   f64 now = vlib_time_now (vm);
3725   nat_outside_fib_t *outside_fib;
3726   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3727   fib_prefix_t pfx = {
3728     .fp_proto = FIB_PROTOCOL_IP4,
3729     .fp_len = 32,
3730     .fp_addr = {
3731                 .ip4.as_u32 = eh_addr->as_u32,
3732                 },
3733   };
3734
3735   key.addr.as_u32 = out_addr->as_u32;
3736   key.port = out_port;
3737   key.protocol = proto;
3738
3739   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3740     {
3741       if (nat_set_outside_address_and_port
3742           (sm->addresses, thread_index, &key))
3743         return;
3744     }
3745
3746   key.addr.as_u32 = ehn_addr->as_u32;
3747   key.port = ehn_port;
3748   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3749     {
3750       if (nat_set_outside_address_and_port
3751           (sm->twice_nat_addresses, thread_index, &key))
3752         return;
3753     }
3754
3755   s = nat_ed_session_alloc (sm, thread_index, now, proto);
3756   if (!s)
3757     return;
3758
3759   s->last_heard = now;
3760   s->flags = flags;
3761   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3762   s->ext_host_nat_port = s->ext_host_port = eh_port;
3763   if (is_twice_nat_session (s))
3764     {
3765       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3766       s->ext_host_nat_port = ehn_port;
3767     }
3768   switch (vec_len (sm->outside_fibs))
3769     {
3770     case 0:
3771       key.fib_index = sm->outside_fib_index;
3772       break;
3773     case 1:
3774       key.fib_index = sm->outside_fibs[0].fib_index;
3775       break;
3776     default:
3777       /* *INDENT-OFF* */
3778       vec_foreach (outside_fib, sm->outside_fibs)
3779         {
3780           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3781           if (FIB_NODE_INDEX_INVALID != fei)
3782             {
3783               if (fib_entry_get_resolving_interface (fei) != ~0)
3784                 {
3785                   key.fib_index = outside_fib->fib_index;
3786                   break;
3787                 }
3788             }
3789         }
3790       /* *INDENT-ON* */
3791       break;
3792     }
3793   key.addr.as_u32 = out_addr->as_u32;
3794   key.port = out_port;
3795   s->out2in = key;
3796   kv.value = s - tsm->sessions;
3797
3798   key.addr.as_u32 = in_addr->as_u32;
3799   key.port = in_port;
3800   key.fib_index = fib_index;
3801   s->in2out = key;
3802
3803   make_ed_kv (in_addr, &s->ext_host_nat_addr,
3804               nat_proto_to_ip_proto (proto), fib_index, in_port,
3805               s->ext_host_nat_port, s - tsm->sessions, &kv);
3806   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3807     nat_elog_warn ("in2out key add failed");
3808
3809   make_ed_kv (out_addr, eh_addr, nat_proto_to_ip_proto (proto),
3810               s->out2in.fib_index, out_port, eh_port, s - tsm->sessions, &kv);
3811   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3812     nat_elog_warn ("out2in key add failed");
3813 }
3814
3815 void
3816 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3817                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3818                    u32 fib_index, u32 ti)
3819 {
3820   snat_main_t *sm = &snat_main;
3821   clib_bihash_kv_16_8_t kv, value;
3822   u32 thread_index;
3823   snat_session_t *s;
3824   snat_main_per_thread_data_t *tsm;
3825
3826   if (sm->num_workers > 1)
3827     thread_index =
3828       sm->first_worker_index +
3829       (sm->workers[(clib_net_to_host_u16 (out_port) -
3830                     1024) / sm->port_per_thread]);
3831   else
3832     thread_index = sm->num_workers;
3833   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3834
3835   make_ed_kv (out_addr, eh_addr, proto, fib_index, out_port, eh_port, ~0ULL,
3836               &kv);
3837   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3838     return;
3839
3840   s = pool_elt_at_index (tsm->sessions, value.value);
3841   nat_free_session_data (sm, s, thread_index, 1);
3842   nat44_delete_session (sm, s, thread_index);
3843 }
3844
3845 void
3846 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3847                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3848                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3849                    u32 thread_index)
3850 {
3851   snat_main_t *sm = &snat_main;
3852   clib_bihash_kv_16_8_t kv, value;
3853   snat_session_t *s;
3854   snat_main_per_thread_data_t *tsm;
3855
3856   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3857
3858   make_ed_kv (out_addr, eh_addr, proto, fib_index, out_port, eh_port, ~0ULL,
3859               &kv);
3860   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3861     return;
3862
3863   s = pool_elt_at_index (tsm->sessions, value.value);
3864   s->total_pkts = total_pkts;
3865   s->total_bytes = total_bytes;
3866 }
3867
3868 void
3869 nat44_db_init (snat_main_per_thread_data_t * tsm)
3870 {
3871   snat_main_t *sm = &snat_main;
3872
3873   pool_alloc (tsm->sessions, sm->max_translations);
3874   pool_alloc (tsm->lru_pool, sm->max_translations);
3875
3876   dlist_elt_t *head;
3877
3878   pool_get (tsm->lru_pool, head);
3879   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3880   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3881
3882   pool_get (tsm->lru_pool, head);
3883   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3884   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3885
3886   pool_get (tsm->lru_pool, head);
3887   tsm->udp_lru_head_index = head - tsm->lru_pool;
3888   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3889
3890   pool_get (tsm->lru_pool, head);
3891   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3892   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3893
3894   pool_get (tsm->lru_pool, head);
3895   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3896   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3897
3898   if (sm->endpoint_dependent)
3899     {
3900       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3901                              sm->translation_buckets,
3902                              sm->translation_memory_size);
3903       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3904                                           format_ed_session_kvp);
3905       clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3906                              sm->translation_buckets,
3907                              sm->translation_memory_size);
3908       clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3909                                           format_ed_session_kvp);
3910     }
3911   else
3912     {
3913       clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3914                             sm->translation_buckets,
3915                             sm->translation_memory_size);
3916       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
3917       clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3918                             sm->translation_buckets,
3919                             sm->translation_memory_size);
3920       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
3921     }
3922
3923   // TODO: resolve static mappings (put only to !ED)
3924   pool_alloc (tsm->list_pool, sm->max_translations);
3925   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets,
3926                         sm->user_memory_size);
3927   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3928 }
3929
3930 void
3931 nat44_db_free (snat_main_per_thread_data_t * tsm)
3932 {
3933   snat_main_t *sm = &snat_main;
3934
3935   pool_free (tsm->sessions);
3936   pool_free (tsm->lru_pool);
3937
3938   if (sm->endpoint_dependent)
3939     {
3940       clib_bihash_free_16_8 (&tsm->in2out_ed);
3941       clib_bihash_free_16_8 (&tsm->out2in_ed);
3942     }
3943   else
3944     {
3945       clib_bihash_free_8_8 (&tsm->in2out);
3946       clib_bihash_free_8_8 (&tsm->out2in);
3947     }
3948
3949   // TODO: resolve static mappings (put only to !ED)
3950   pool_free (tsm->users);
3951   pool_free (tsm->list_pool);
3952   clib_bihash_free_8_8 (&tsm->user_hash);
3953 }
3954
3955 static clib_error_t *
3956 snat_config (vlib_main_t * vm, unformat_input_t * input)
3957 {
3958   snat_main_t *sm = &snat_main;
3959   nat66_main_t *nm = &nat66_main;
3960   snat_main_per_thread_data_t *tsm;
3961
3962   u32 static_mapping_buckets = 1024;
3963   uword static_mapping_memory_size = 64 << 20;
3964
3965   u32 nat64_bib_buckets = 1024;
3966   u32 nat64_bib_memory_size = 128 << 20;
3967
3968   u32 nat64_st_buckets = 2048;
3969   uword nat64_st_memory_size = 256 << 20;
3970
3971   u32 user_buckets = 128;
3972   uword user_memory_size = 64 << 20;
3973   u32 translation_buckets = 1024;
3974   uword translation_memory_size = 128 << 20;
3975
3976   u32 max_translations_per_user = ~0;
3977
3978   u32 outside_vrf_id = 0;
3979   u32 outside_ip6_vrf_id = 0;
3980   u32 inside_vrf_id = 0;
3981   u8 static_mapping_only = 0;
3982   u8 static_mapping_connection_tracking = 0;
3983
3984   // configurable timeouts
3985   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3986   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3987   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3988   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3989
3990   sm->deterministic = 0;
3991   sm->out2in_dpo = 0;
3992   sm->endpoint_dependent = 0;
3993
3994   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3995     {
3996       if (unformat
3997           (input, "translation hash buckets %d", &translation_buckets))
3998         ;
3999       else if (unformat (input, "udp timeout %d", &udp_timeout))
4000         ;
4001       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
4002         ;
4003       else if (unformat (input, "tcp transitory timeout %d",
4004                          &tcp_transitory_timeout));
4005       else if (unformat (input, "tcp established timeout %d",
4006                          &tcp_established_timeout));
4007       else if (unformat (input, "translation hash memory %d",
4008                          &translation_memory_size));
4009       else if (unformat (input, "user hash buckets %d", &user_buckets))
4010         ;
4011       else if (unformat (input, "user hash memory %d", &user_memory_size))
4012         ;
4013       else if (unformat (input, "max translations per user %d",
4014                          &max_translations_per_user))
4015         ;
4016       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
4017         ;
4018       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
4019         ;
4020       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
4021         ;
4022       else if (unformat (input, "static mapping only"))
4023         {
4024           static_mapping_only = 1;
4025           if (unformat (input, "connection tracking"))
4026             static_mapping_connection_tracking = 1;
4027         }
4028       else if (unformat (input, "deterministic"))
4029         sm->deterministic = 1;
4030       else if (unformat (input, "nat64 bib hash buckets %d",
4031                          &nat64_bib_buckets))
4032         ;
4033       else if (unformat (input, "nat64 bib hash memory %d",
4034                          &nat64_bib_memory_size))
4035         ;
4036       else
4037         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
4038         ;
4039       else if (unformat (input, "nat64 st hash memory %d",
4040                          &nat64_st_memory_size))
4041         ;
4042       else if (unformat (input, "out2in dpo"))
4043         sm->out2in_dpo = 1;
4044       //else if (unformat (input, "dslite ce"))
4045       //dslite_set_ce (dm, 1);
4046       else if (unformat (input, "endpoint-dependent"))
4047         sm->endpoint_dependent = 1;
4048       else
4049         return clib_error_return (0, "unknown input '%U'",
4050                                   format_unformat_error, input);
4051     }
4052
4053   if (sm->deterministic && sm->endpoint_dependent)
4054     return clib_error_return (0,
4055                               "deterministic and endpoint-dependent modes are mutually exclusive");
4056
4057   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
4058     return clib_error_return (0,
4059                               "static mapping only mode available only for simple nat");
4060
4061   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
4062     return clib_error_return (0,
4063                               "out2in dpo mode available only for simple nat");
4064
4065   /* optionally configurable timeouts for testing purposes */
4066   sm->udp_timeout = udp_timeout;
4067   sm->tcp_transitory_timeout = tcp_transitory_timeout;
4068   sm->tcp_established_timeout = tcp_established_timeout;
4069   sm->icmp_timeout = icmp_timeout;
4070
4071   sm->user_buckets = user_buckets;
4072   sm->user_memory_size = user_memory_size;
4073
4074   sm->translation_buckets = translation_buckets;
4075   sm->translation_memory_size = translation_memory_size;
4076   /* do not exceed load factor 10 */
4077   sm->max_translations = 10 * translation_buckets;
4078   vec_add1 (sm->max_translations_per_fib, sm->max_translations);
4079
4080   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4081     sm->max_translations : max_translations_per_user;
4082
4083   sm->outside_vrf_id = outside_vrf_id;
4084   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4085                                                              outside_vrf_id,
4086                                                              nat_fib_src_hi);
4087   nm->outside_vrf_id = outside_ip6_vrf_id;
4088   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
4089                                                              outside_ip6_vrf_id,
4090                                                              nat_fib_src_hi);
4091   sm->inside_vrf_id = inside_vrf_id;
4092   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4093                                                             inside_vrf_id,
4094                                                             nat_fib_src_hi);
4095   sm->static_mapping_only = static_mapping_only;
4096   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4097
4098   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4099                   nat64_st_memory_size);
4100
4101   if (sm->deterministic)
4102     {
4103       sm->in2out_node_index = snat_det_in2out_node.index;
4104       sm->in2out_output_node_index = ~0;
4105       sm->out2in_node_index = snat_det_out2in_node.index;
4106       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4107       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4108     }
4109   else
4110     {
4111       if (sm->endpoint_dependent)
4112         {
4113           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4114           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4115
4116           sm->in2out_node_index = nat44_ed_in2out_node.index;
4117           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4118           sm->out2in_node_index = nat44_ed_out2in_node.index;
4119
4120           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4121           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4122           nat_affinity_init (vm);
4123           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4124                        nat_ha_sref_ed_cb);
4125         }
4126       else
4127         {
4128           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4129           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4130
4131           sm->in2out_node_index = snat_in2out_node.index;
4132           sm->in2out_output_node_index = snat_in2out_output_node.index;
4133           sm->out2in_node_index = snat_out2in_node.index;
4134
4135           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4136           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4137           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4138         }
4139       if (!static_mapping_only ||
4140           (static_mapping_only && static_mapping_connection_tracking))
4141         {
4142           /* *INDENT-OFF* */
4143           vec_foreach (tsm, sm->per_thread_data)
4144             {
4145               nat44_db_init (tsm);
4146             }
4147           /* *INDENT-ON* */
4148         }
4149       else
4150         {
4151           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4152           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4153         }
4154       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4155                             "static_mapping_by_local", static_mapping_buckets,
4156                             static_mapping_memory_size);
4157       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4158                                          format_static_mapping_kvp);
4159
4160       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4161                             "static_mapping_by_external",
4162                             static_mapping_buckets,
4163                             static_mapping_memory_size);
4164       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4165                                          format_static_mapping_kvp);
4166     }
4167
4168   return 0;
4169 }
4170
4171 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4172
4173 static void
4174 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4175                                  uword opaque,
4176                                  u32 sw_if_index,
4177                                  ip4_address_t * address,
4178                                  u32 address_length,
4179                                  u32 if_address_index, u32 is_delete)
4180 {
4181   snat_main_t *sm = &snat_main;
4182   snat_static_map_resolve_t *rp;
4183   snat_static_mapping_t *m;
4184   snat_session_key_t m_key;
4185   clib_bihash_kv_8_8_t kv, value;
4186   int i, rv;
4187   ip4_address_t l_addr;
4188
4189   for (i = 0; i < vec_len (sm->to_resolve); i++)
4190     {
4191       rp = sm->to_resolve + i;
4192       if (rp->addr_only == 0)
4193         continue;
4194       if (rp->sw_if_index == sw_if_index)
4195         goto match;
4196     }
4197
4198   return;
4199
4200 match:
4201   m_key.addr.as_u32 = address->as_u32;
4202   m_key.port = rp->addr_only ? 0 : rp->e_port;
4203   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4204   m_key.fib_index = sm->outside_fib_index;
4205   kv.key = m_key.as_u64;
4206   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4207     m = 0;
4208   else
4209     m = pool_elt_at_index (sm->static_mappings, value.value);
4210
4211   if (!is_delete)
4212     {
4213       /* Don't trip over lease renewal, static config */
4214       if (m)
4215         return;
4216     }
4217   else
4218     {
4219       if (!m)
4220         return;
4221     }
4222
4223   /* Indetity mapping? */
4224   if (rp->l_addr.as_u32 == 0)
4225     l_addr.as_u32 = address[0].as_u32;
4226   else
4227     l_addr.as_u32 = rp->l_addr.as_u32;
4228   /* Add the static mapping */
4229   rv = snat_add_static_mapping (l_addr,
4230                                 address[0],
4231                                 rp->l_port,
4232                                 rp->e_port,
4233                                 rp->vrf_id,
4234                                 rp->addr_only, ~0 /* sw_if_index */ ,
4235                                 rp->proto, !is_delete, rp->twice_nat,
4236                                 rp->out2in_only, rp->tag, rp->identity_nat);
4237   if (rv)
4238     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4239 }
4240
4241 static void
4242 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4243                                        uword opaque,
4244                                        u32 sw_if_index,
4245                                        ip4_address_t * address,
4246                                        u32 address_length,
4247                                        u32 if_address_index, u32 is_delete)
4248 {
4249   snat_main_t *sm = &snat_main;
4250   snat_static_map_resolve_t *rp;
4251   ip4_address_t l_addr;
4252   int i, j;
4253   int rv;
4254   u8 twice_nat = 0;
4255   snat_address_t *addresses = sm->addresses;
4256
4257   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4258     {
4259       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4260         goto match;
4261     }
4262
4263   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4264     {
4265       twice_nat = 1;
4266       addresses = sm->twice_nat_addresses;
4267       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4268         goto match;
4269     }
4270
4271   return;
4272
4273 match:
4274   if (!is_delete)
4275     {
4276       /* Don't trip over lease renewal, static config */
4277       for (j = 0; j < vec_len (addresses); j++)
4278         if (addresses[j].addr.as_u32 == address->as_u32)
4279           return;
4280
4281       (void) snat_add_address (sm, address, ~0, twice_nat);
4282       /* Scan static map resolution vector */
4283       for (j = 0; j < vec_len (sm->to_resolve); j++)
4284         {
4285           rp = sm->to_resolve + j;
4286           if (rp->addr_only)
4287             continue;
4288           /* On this interface? */
4289           if (rp->sw_if_index == sw_if_index)
4290             {
4291               /* Indetity mapping? */
4292               if (rp->l_addr.as_u32 == 0)
4293                 l_addr.as_u32 = address[0].as_u32;
4294               else
4295                 l_addr.as_u32 = rp->l_addr.as_u32;
4296               /* Add the static mapping */
4297               rv = snat_add_static_mapping (l_addr,
4298                                             address[0],
4299                                             rp->l_port,
4300                                             rp->e_port,
4301                                             rp->vrf_id,
4302                                             rp->addr_only,
4303                                             ~0 /* sw_if_index */ ,
4304                                             rp->proto,
4305                                             rp->is_add, rp->twice_nat,
4306                                             rp->out2in_only, rp->tag,
4307                                             rp->identity_nat);
4308               if (rv)
4309                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4310                                     "i4", rv);
4311             }
4312         }
4313       return;
4314     }
4315   else
4316     {
4317       (void) snat_del_address (sm, address[0], 1, twice_nat);
4318       return;
4319     }
4320 }
4321
4322
4323 int
4324 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4325                             u8 twice_nat)
4326 {
4327   ip4_main_t *ip4_main = sm->ip4_main;
4328   ip4_address_t *first_int_addr;
4329   snat_static_map_resolve_t *rp;
4330   u32 *indices_to_delete = 0;
4331   int i, j;
4332   u32 *auto_add_sw_if_indices =
4333     twice_nat ? sm->
4334     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4335
4336   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4337     );
4338
4339   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4340     {
4341       if (auto_add_sw_if_indices[i] == sw_if_index)
4342         {
4343           if (is_del)
4344             {
4345               /* if have address remove it */
4346               if (first_int_addr)
4347                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4348               else
4349                 {
4350                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4351                     {
4352                       rp = sm->to_resolve + j;
4353                       if (rp->sw_if_index == sw_if_index)
4354                         vec_add1 (indices_to_delete, j);
4355                     }
4356                   if (vec_len (indices_to_delete))
4357                     {
4358                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4359                         vec_del1 (sm->to_resolve, j);
4360                       vec_free (indices_to_delete);
4361                     }
4362                 }
4363               if (twice_nat)
4364                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4365               else
4366                 vec_del1 (sm->auto_add_sw_if_indices, i);
4367             }
4368           else
4369             return VNET_API_ERROR_VALUE_EXIST;
4370
4371           return 0;
4372         }
4373     }
4374
4375   if (is_del)
4376     return VNET_API_ERROR_NO_SUCH_ENTRY;
4377
4378   /* add to the auto-address list */
4379   if (twice_nat)
4380     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4381   else
4382     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4383
4384   /* If the address is already bound - or static - add it now */
4385   if (first_int_addr)
4386     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4387
4388   return 0;
4389 }
4390
4391 int
4392 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4393                    nat_protocol_t proto, u32 vrf_id, int is_in)
4394 {
4395   snat_main_per_thread_data_t *tsm;
4396   clib_bihash_kv_8_8_t kv, value;
4397   ip4_header_t ip;
4398   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4399   snat_session_key_t key;
4400   snat_session_t *s;
4401   clib_bihash_8_8_t *t;
4402
4403   if (sm->endpoint_dependent)
4404     return VNET_API_ERROR_UNSUPPORTED;
4405
4406   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4407   if (sm->num_workers > 1)
4408     tsm =
4409       vec_elt_at_index (sm->per_thread_data,
4410                         sm->worker_in2out_cb (&ip, fib_index, 0));
4411   else
4412     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4413
4414   key.addr.as_u32 = addr->as_u32;
4415   key.port = clib_host_to_net_u16 (port);
4416   key.protocol = proto;
4417   key.fib_index = fib_index;
4418   kv.key = key.as_u64;
4419   t = is_in ? &tsm->in2out : &tsm->out2in;
4420   if (!clib_bihash_search_8_8 (t, &kv, &value))
4421     {
4422       if (pool_is_free_index (tsm->sessions, value.value))
4423         return VNET_API_ERROR_UNSPECIFIED;
4424
4425       s = pool_elt_at_index (tsm->sessions, value.value);
4426       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4427       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4428       return 0;
4429     }
4430
4431   return VNET_API_ERROR_NO_SUCH_ENTRY;
4432 }
4433
4434 int
4435 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4436                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4437                       u32 vrf_id, int is_in)
4438 {
4439   ip4_header_t ip;
4440   clib_bihash_16_8_t *t;
4441   clib_bihash_kv_16_8_t kv, value;
4442   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4443   snat_session_t *s;
4444   snat_main_per_thread_data_t *tsm;
4445
4446   if (!sm->endpoint_dependent)
4447     return VNET_API_ERROR_FEATURE_DISABLED;
4448
4449   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4450   if (sm->num_workers > 1)
4451     tsm =
4452       vec_elt_at_index (sm->per_thread_data,
4453                         sm->worker_in2out_cb (&ip, fib_index, 0));
4454   else
4455     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4456
4457   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4458   make_ed_kv (addr, eh_addr, proto, fib_index, clib_host_to_net_u16 (port),
4459               clib_host_to_net_u16 (eh_port), ~0ULL, &kv);
4460   if (clib_bihash_search_16_8 (t, &kv, &value))
4461     {
4462       return VNET_API_ERROR_NO_SUCH_ENTRY;
4463     }
4464
4465   if (pool_is_free_index (tsm->sessions, value.value))
4466     return VNET_API_ERROR_UNSPECIFIED;
4467   s = pool_elt_at_index (tsm->sessions, value.value);
4468   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4469   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
4470   return 0;
4471 }
4472
4473 void
4474 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4475 {
4476   snat_main_t *sm = &snat_main;
4477
4478   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4479   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4480   sm->psid = psid;
4481   sm->psid_offset = psid_offset;
4482   sm->psid_length = psid_length;
4483 }
4484
4485 void
4486 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4487 {
4488   snat_main_t *sm = &snat_main;
4489
4490   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4491   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4492   sm->start_port = start_port;
4493   sm->end_port = end_port;
4494 }
4495
4496 void
4497 nat_set_alloc_addr_and_port_default (void)
4498 {
4499   snat_main_t *sm = &snat_main;
4500
4501   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4502   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4503 }
4504
4505 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4506                                  vlib_node_runtime_t * node,
4507                                  vlib_frame_t * frame)
4508 {
4509   return 0;
4510 }
4511
4512 /* *INDENT-OFF* */
4513 VLIB_REGISTER_NODE (nat_default_node) = {
4514   .name = "nat-default",
4515   .vector_size = sizeof (u32),
4516   .format_trace = 0,
4517   .type = VLIB_NODE_TYPE_INTERNAL,
4518   .n_errors = 0,
4519   .n_next_nodes = NAT_N_NEXT,
4520   .next_nodes = {
4521     [NAT_NEXT_DROP] = "error-drop",
4522     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4523     //[NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4524     //[NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4525     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4526     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4527     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4528     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4529     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4530     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4531     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4532   },
4533 };
4534 /* *INDENT-ON* */
4535
4536 /*
4537  * fd.io coding-style-patch-verification: ON
4538  *
4539  * Local Variables:
4540  * eval: (c-set-style "gnu")
4541  * End:
4542  */