src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36 #include <vppinfra/bihash_16_8.h>
  37
  38 #include <vpp/app/version.h>
  39
  40 snat_main_t snat_main;
  41
  42 fib_source_t nat_fib_src_hi;
  43 fib_source_t nat_fib_src_low;
  44
  45 /* *INDENT-OFF* */
  46 /* Hook up input features */
  47 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  48   .arc_name = "ip4-unicast",
  49   .node_name = "nat-pre-in2out",
  50   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  51                                "ip4-sv-reassembly-feature"),
  52 };
  53 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  54   .arc_name = "ip4-unicast",
  55   .node_name = "nat-pre-out2in",
  56   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  57                                "ip4-dhcp-client-detect",
  58                                "ip4-sv-reassembly-feature"),
  59 };
  60 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  61   .arc_name = "ip4-unicast",
  62   .node_name = "nat44-in2out-worker-handoff",
  63   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  64 };
  65 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  66   .arc_name = "ip4-unicast",
  67   .node_name = "nat44-out2in-worker-handoff",
  68   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  69                                "ip4-dhcp-client-detect"),
  70 };
  71 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  72   .arc_name = "ip4-unicast",
  73   .node_name = "nat44-in2out",
  74   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  75 };
  76 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  77   .arc_name = "ip4-unicast",
  78   .node_name = "nat44-out2in",
  79   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  80                                "ip4-dhcp-client-detect"),
  81 };
  82 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  83   .arc_name = "ip4-unicast",
  84   .node_name = "nat44-classify",
  85   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  86 };
  87 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  88   .arc_name = "ip4-unicast",
  89   .node_name = "nat44-det-in2out",
  90   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-det-out2in",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  96                                "ip4-dhcp-client-detect"),
  97 };
  98 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-det-classify",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 102 };
 103 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-ed-in2out",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-ed-out2in",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 112                                "ip4-dhcp-client-detect"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-ed-classify",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 118 };
 119 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-handoff-classify",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 123 };
 124 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-in2out-fast",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 128 };
 129 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 130   .arc_name = "ip4-unicast",
 131   .node_name = "nat44-out2in-fast",
 132   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 133                                "ip4-dhcp-client-detect"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 136   .arc_name = "ip4-unicast",
 137   .node_name = "nat44-hairpin-dst",
 138   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 141   .arc_name = "ip4-unicast",
 142   .node_name = "nat44-ed-hairpin-dst",
 143   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 144 };
 145
 146 /* Hook up output features */
 147 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 148   .arc_name = "ip4-output",
 149   .node_name = "nat44-in2out-output",
 150   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 151 };
 152 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 153   .arc_name = "ip4-output",
 154   .node_name = "nat44-in2out-output-worker-handoff",
 155   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 156 };
 157 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 158   .arc_name = "ip4-output",
 159   .node_name = "nat44-hairpin-src",
 160   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 161 };
 162 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 163   .arc_name = "ip4-output",
 164   .node_name = "nat44-ed-in2out-output",
 165   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 166   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 167 };
 168 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 169   .arc_name = "ip4-output",
 170   .node_name = "nat44-ed-hairpin-src",
 171   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 172   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 173 };
 174
 175 /* Hook up ip4-local features */
 176 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 177 {
 178   .arc_name = "ip4-local",
 179   .node_name = "nat44-hairpinning",
 180   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 181 };
 182 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 183 {
 184   .arc_name = "ip4-local",
 185   .node_name = "nat44-ed-hairpinning",
 186   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 187 };
 188
 189
 190 VLIB_PLUGIN_REGISTER () = {
 191     .version = VPP_BUILD_VER,
 192     .description = "Network Address Translation (NAT)",
 193 };
 194 /* *INDENT-ON* */
 195
 196 void
 197 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 198                        u8 is_ha)
 199 {
 200   snat_session_key_t key;
 201   clib_bihash_kv_8_8_t kv;
 202   u8 proto;
 203   u16 r_port, l_port;
 204   ip4_address_t *l_addr, *r_addr;
 205   u32 fib_index = 0;
 206   clib_bihash_kv_16_8_t ed_kv;
 207   snat_main_per_thread_data_t *tsm =
 208     vec_elt_at_index (sm->per_thread_data, thread_index);
 209
 210   if (is_fwd_bypass_session (s))
 211     {
 212       if (snat_is_unk_proto_session (s))
 213         {
 214           make_ed_kv (&s->in2out.addr, &s->ext_host_addr, s->in2out.port, 0,
 215                       0, 0, ~0ULL, &ed_kv);
 216         }
 217       else
 218         {
 219           l_port = s->in2out.port;
 220           r_port = s->ext_host_port;
 221           l_addr = &s->in2out.addr;
 222           r_addr = &s->ext_host_addr;
 223           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 224           make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 225                       &ed_kv);
 226         }
 227       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 228         nat_elog_warn ("in2out_ed key del failed");
 229       return;
 230     }
 231
 232   /* session lookup tables */
 233   if (is_ed_session (s))
 234     {
 235       if (is_affinity_sessions (s))
 236         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 237                              s->in2out.protocol, s->out2in.port);
 238       l_addr = &s->out2in.addr;
 239       r_addr = &s->ext_host_addr;
 240       fib_index = s->out2in.fib_index;
 241       if (snat_is_unk_proto_session (s))
 242         {
 243           proto = s->in2out.port;
 244           r_port = 0;
 245           l_port = 0;
 246         }
 247       else
 248         {
 249           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 250           l_port = s->out2in.port;
 251           r_port = s->ext_host_port;
 252         }
 253       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 254                   &ed_kv);
 255       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 256         nat_elog_warn ("out2in_ed key del failed");
 257       l_addr = &s->in2out.addr;
 258       fib_index = s->in2out.fib_index;
 259       if (!snat_is_unk_proto_session (s))
 260         l_port = s->in2out.port;
 261       if (is_twice_nat_session (s))
 262         {
 263           r_addr = &s->ext_host_nat_addr;
 264           r_port = s->ext_host_nat_port;
 265         }
 266       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 267                   &ed_kv);
 268       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 269         nat_elog_warn ("in2out_ed key del failed");
 270
 271       if (!is_ha)
 272         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 273                                &s->in2out.addr, s->in2out.port,
 274                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 275                                &s->out2in.addr, s->out2in.port,
 276                                &s->ext_host_addr, s->ext_host_port,
 277                                s->in2out.protocol, is_twice_nat_session (s));
 278     }
 279   else
 280     {
 281       kv.key = s->in2out.as_u64;
 282       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 283         nat_elog_warn ("in2out key del failed");
 284       kv.key = s->out2in.as_u64;
 285       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 286         nat_elog_warn ("out2in key del failed");
 287
 288       if (!is_ha)
 289         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 290                                  &s->in2out.addr, s->in2out.port,
 291                                  &s->out2in.addr, s->out2in.port,
 292                                  s->in2out.protocol);
 293     }
 294
 295   if (snat_is_unk_proto_session (s))
 296     return;
 297
 298   if (!is_ha)
 299     {
 300       /* log NAT event */
 301       snat_ipfix_logging_nat44_ses_delete (thread_index,
 302                                            s->in2out.addr.as_u32,
 303                                            s->out2in.addr.as_u32,
 304                                            s->in2out.protocol,
 305                                            s->in2out.port,
 306                                            s->out2in.port,
 307                                            s->in2out.fib_index);
 308
 309       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 310                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 311                    thread_index);
 312     }
 313
 314   /* Twice NAT address and port for external host */
 315   if (is_twice_nat_session (s))
 316     {
 317       key.protocol = s->in2out.protocol;
 318       key.port = s->ext_host_nat_port;
 319       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 320       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 321                                           thread_index, &key);
 322     }
 323
 324   if (snat_is_session_static (s))
 325     return;
 326
 327   snat_free_outside_address_and_port (sm->addresses, thread_index,
 328                                       &s->out2in);
 329 }
 330
 331 int
 332 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
 333 {
 334   snat_main_t *sm = &snat_main;
 335   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
 336   u32 len = vec_len (sm->max_translations_per_fib);
 337
 338   if (len <= fib_index)
 339     {
 340       vec_validate (sm->max_translations_per_fib, fib_index + 1);
 341
 342       for (; len < vec_len (sm->max_translations_per_fib); len++)
 343         sm->max_translations_per_fib[len] = sm->max_translations;
 344     }
 345
 346   sm->max_translations_per_fib[fib_index] = session_limit;
 347   return 0;
 348 }
 349
 350 void
 351 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 352                          u32 thread_index, u8 is_ha)
 353 {
 354   snat_session_key_t key;
 355   u8 proto;
 356   u16 r_port, l_port;
 357   ip4_address_t *l_addr, *r_addr;
 358   u32 fib_index;
 359   clib_bihash_kv_16_8_t ed_kv;
 360   snat_main_per_thread_data_t *tsm =
 361     vec_elt_at_index (sm->per_thread_data, thread_index);
 362
 363   if (is_fwd_bypass_session (s))
 364     {
 365       if (snat_is_unk_proto_session (s))
 366         {
 367           proto = s->in2out.port;
 368           r_port = 0;
 369           l_port = 0;
 370         }
 371       else
 372         {
 373           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 374           l_port = s->in2out.port;
 375           r_port = s->ext_host_port;
 376         }
 377
 378       l_addr = &s->in2out.addr;
 379       r_addr = &s->ext_host_addr;
 380       fib_index = 0;
 381       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 382                   &ed_kv);
 383
 384       if (PREDICT_FALSE
 385           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 386         nat_elog_warn ("in2out_ed key del failed");
 387       return;
 388     }
 389
 390   /* session lookup tables */
 391   if (is_affinity_sessions (s))
 392     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 393                          s->in2out.protocol, s->out2in.port);
 394   l_addr = &s->out2in.addr;
 395   r_addr = &s->ext_host_addr;
 396   fib_index = s->out2in.fib_index;
 397   if (snat_is_unk_proto_session (s))
 398     {
 399       proto = s->in2out.port;
 400       r_port = 0;
 401       l_port = 0;
 402     }
 403   else
 404     {
 405       proto = snat_proto_to_ip_proto (s->in2out.protocol);
 406       l_port = s->out2in.port;
 407       r_port = s->ext_host_port;
 408     }
 409   make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 410               &ed_kv);
 411
 412   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0)))
 413     nat_elog_warn ("out2in_ed key del failed");
 414
 415   l_addr = &s->in2out.addr;
 416   fib_index = s->in2out.fib_index;
 417
 418   if (!snat_is_unk_proto_session (s))
 419     l_port = s->in2out.port;
 420
 421   if (is_twice_nat_session (s))
 422     {
 423       r_addr = &s->ext_host_nat_addr;
 424       r_port = s->ext_host_nat_port;
 425     }
 426   make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 427               &ed_kv);
 428
 429   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 430     nat_elog_warn ("in2out_ed key del failed");
 431
 432   if (!is_ha)
 433     {
 434       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 435                              &s->in2out.addr, s->in2out.port,
 436                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 437                              &s->out2in.addr, s->out2in.port,
 438                              &s->ext_host_addr, s->ext_host_port,
 439                              s->in2out.protocol, is_twice_nat_session (s));
 440     }
 441
 442   if (snat_is_unk_proto_session (s))
 443     return;
 444
 445   // is this correct ?
 446   if (!is_ha)
 447     {
 448       snat_ipfix_logging_nat44_ses_delete (thread_index,
 449                                            s->in2out.addr.as_u32,
 450                                            s->out2in.addr.as_u32,
 451                                            s->in2out.protocol,
 452                                            s->in2out.port,
 453                                            s->out2in.port,
 454                                            s->in2out.fib_index);
 455       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 456                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 457                    thread_index);
 458     }
 459
 460   /* Twice NAT address and port for external host */
 461   if (is_twice_nat_session (s))
 462     {
 463       key.protocol = s->in2out.protocol;
 464       key.port = s->ext_host_nat_port;
 465       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 466       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 467                                           thread_index, &key);
 468     }
 469
 470   if (snat_is_session_static (s))
 471     return;
 472
 473   // should be called for every dynamic session
 474   snat_free_outside_address_and_port (sm->addresses, thread_index,
 475                                       &s->out2in);
 476 }
 477
 478
 479 snat_user_t *
 480 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 481                         u32 thread_index)
 482 {
 483   snat_user_t *u = 0;
 484   snat_user_key_t user_key;
 485   clib_bihash_kv_8_8_t kv, value;
 486   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 487   dlist_elt_t *per_user_list_head_elt;
 488
 489   user_key.addr.as_u32 = addr->as_u32;
 490   user_key.fib_index = fib_index;
 491   kv.key = user_key.as_u64;
 492
 493   /* Ever heard of the "user" = src ip4 address before? */
 494   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 495     {
 496       /* no, make a new one */
 497       pool_get (tsm->users, u);
 498       clib_memset (u, 0, sizeof (*u));
 499
 500       u->addr.as_u32 = addr->as_u32;
 501       u->fib_index = fib_index;
 502
 503       pool_get (tsm->list_pool, per_user_list_head_elt);
 504
 505       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 506         tsm->list_pool;
 507
 508       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 509
 510       kv.value = u - tsm->users;
 511
 512       /* add user */
 513       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 514         {
 515           nat_elog_warn ("user_hash key add failed");
 516           nat44_delete_user_with_no_session (sm, u, thread_index);
 517           return NULL;
 518         }
 519
 520       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 521                                pool_elts (tsm->users));
 522     }
 523   else
 524     {
 525       u = pool_elt_at_index (tsm->users, value.value);
 526     }
 527
 528   return u;
 529 }
 530
 531 snat_session_t *
 532 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 533                               u32 thread_index, f64 now)
 534 {
 535   snat_session_t *s;
 536   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 537   u32 oldest_per_user_translation_list_index, session_index;
 538   dlist_elt_t *oldest_per_user_translation_list_elt;
 539   dlist_elt_t *per_user_translation_list_elt;
 540
 541   /* Over quota? Recycle the least recently used translation */
 542   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 543     {
 544       oldest_per_user_translation_list_index =
 545         clib_dlist_remove_head (tsm->list_pool,
 546                                 u->sessions_per_user_list_head_index);
 547
 548       ASSERT (oldest_per_user_translation_list_index != ~0);
 549
 550       /* Add it back to the end of the LRU list */
 551       clib_dlist_addtail (tsm->list_pool,
 552                           u->sessions_per_user_list_head_index,
 553                           oldest_per_user_translation_list_index);
 554       /* Get the list element */
 555       oldest_per_user_translation_list_elt =
 556         pool_elt_at_index (tsm->list_pool,
 557                            oldest_per_user_translation_list_index);
 558
 559       /* Get the session index from the list element */
 560       session_index = oldest_per_user_translation_list_elt->value;
 561
 562       /* Get the session */
 563       s = pool_elt_at_index (tsm->sessions, session_index);
 564       nat_free_session_data (sm, s, thread_index, 0);
 565       if (snat_is_session_static (s))
 566         u->nstaticsessions--;
 567       else
 568         u->nsessions--;
 569       s->flags = 0;
 570       s->total_bytes = 0;
 571       s->total_pkts = 0;
 572       s->state = 0;
 573       s->ext_host_addr.as_u32 = 0;
 574       s->ext_host_port = 0;
 575       s->ext_host_nat_addr.as_u32 = 0;
 576       s->ext_host_nat_port = 0;
 577     }
 578   else
 579     {
 580       pool_get (tsm->sessions, s);
 581       clib_memset (s, 0, sizeof (*s));
 582
 583       /* Create list elts */
 584       pool_get (tsm->list_pool, per_user_translation_list_elt);
 585       clib_dlist_init (tsm->list_pool,
 586                        per_user_translation_list_elt - tsm->list_pool);
 587
 588       per_user_translation_list_elt->value = s - tsm->sessions;
 589       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 590       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 591
 592       clib_dlist_addtail (tsm->list_pool,
 593                           s->per_user_list_head_index,
 594                           per_user_translation_list_elt - tsm->list_pool);
 595
 596       dlist_elt_t *global_lru_list_elt;
 597       pool_get (tsm->global_lru_pool, global_lru_list_elt);
 598       global_lru_list_elt->value = s - tsm->sessions;
 599       s->global_lru_index = global_lru_list_elt - tsm->global_lru_pool;
 600       clib_dlist_addtail (tsm->global_lru_pool, tsm->global_lru_head_index,
 601                           s->global_lru_index);
 602       s->last_lru_update = now;
 603
 604       s->user_index = u - tsm->users;
 605       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 606                                pool_elts (tsm->sessions));
 607     }
 608
 609   s->ha_last_refreshed = now;
 610
 611   return s;
 612 }
 613
 614 int
 615 nat_global_lru_free_one (snat_main_t * sm, int thread_index, f64 now)
 616 {
 617   snat_session_t *s = NULL;
 618   dlist_elt_t *oldest_elt;
 619   u64 sess_timeout_time;
 620   u32 oldest_index;
 621   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 622   oldest_index = clib_dlist_remove_head (tsm->global_lru_pool,
 623                                          tsm->global_lru_head_index);
 624   if (~0 != oldest_index)
 625     {
 626       oldest_elt = pool_elt_at_index (tsm->global_lru_pool, oldest_index);
 627       s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 628
 629       sess_timeout_time =
 630         s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 631       if (now >= sess_timeout_time
 632           || (s->tcp_close_timestamp && now >= s->tcp_close_timestamp))
 633         {
 634           nat_free_session_data (sm, s, thread_index, 0);
 635           nat44_ed_delete_session (sm, s, thread_index, 0);
 636           return 1;
 637         }
 638       else
 639         {
 640           clib_dlist_addhead (tsm->global_lru_pool,
 641                               tsm->global_lru_head_index, oldest_index);
 642         }
 643     }
 644   return 0;
 645 }
 646
 647 snat_session_t *
 648 nat_ed_session_alloc (snat_main_t * sm, u32 thread_index, f64 now)
 649 {
 650   snat_session_t *s;
 651   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 652
 653   nat_global_lru_free_one (sm, thread_index, now);
 654
 655   pool_get (tsm->sessions, s);
 656   clib_memset (s, 0, sizeof (*s));
 657
 658   nat44_global_lru_insert (tsm, s, now);
 659
 660   s->ha_last_refreshed = now;
 661   vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 662                            pool_elts (tsm->sessions));
 663   return s;
 664 }
 665
 666 void
 667 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 668                           int is_add)
 669 {
 670   fib_prefix_t prefix = {
 671     .fp_len = p_len,
 672     .fp_proto = FIB_PROTOCOL_IP4,
 673     .fp_addr = {
 674                 .ip4.as_u32 = addr->as_u32,
 675                 },
 676   };
 677   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 678
 679   if (is_add)
 680     fib_table_entry_update_one_path (fib_index,
 681                                      &prefix,
 682                                      nat_fib_src_low,
 683                                      (FIB_ENTRY_FLAG_CONNECTED |
 684                                       FIB_ENTRY_FLAG_LOCAL |
 685                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 686                                      DPO_PROTO_IP4,
 687                                      NULL,
 688                                      sw_if_index,
 689                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 690   else
 691     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 692 }
 693
 694 int
 695 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 696                   u8 twice_nat)
 697 {
 698   snat_address_t *ap;
 699   snat_interface_t *i;
 700   vlib_thread_main_t *tm = vlib_get_thread_main ();
 701
 702   if (twice_nat && !sm->endpoint_dependent)
 703     return VNET_API_ERROR_FEATURE_DISABLED;
 704
 705   /* Check if address already exists */
 706   /* *INDENT-OFF* */
 707   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 708     {
 709       if (ap->addr.as_u32 == addr->as_u32)
 710         return VNET_API_ERROR_VALUE_EXIST;
 711     }
 712   /* *INDENT-ON* */
 713
 714   if (twice_nat)
 715     vec_add2 (sm->twice_nat_addresses, ap, 1);
 716   else
 717     vec_add2 (sm->addresses, ap, 1);
 718
 719   ap->addr = *addr;
 720   if (vrf_id != ~0)
 721     ap->fib_index =
 722       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 723                                          nat_fib_src_low);
 724   else
 725     ap->fib_index = ~0;
 726 #define _(N, i, n, s) \
 727   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 728   ap->busy_##n##_ports = 0; \
 729   ap->busy_##n##_ports_per_thread = 0;\
 730   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 731   foreach_snat_protocol
 732 #undef _
 733     if (twice_nat)
 734     return 0;
 735
 736   /* Add external address to FIB */
 737   /* *INDENT-OFF* */
 738   pool_foreach (i, sm->interfaces,
 739   ({
 740     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 741       continue;
 742
 743     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 744     break;
 745   }));
 746   pool_foreach (i, sm->output_feature_interfaces,
 747   ({
 748     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 749       continue;
 750
 751     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 752     break;
 753   }));
 754   /* *INDENT-ON* */
 755
 756   return 0;
 757 }
 758
 759 static int
 760 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 761 {
 762   snat_static_mapping_t *m;
 763   /* *INDENT-OFF* */
 764   pool_foreach (m, sm->static_mappings,
 765   ({
 766       if (is_addr_only_static_mapping (m) ||
 767           is_out2in_only_static_mapping (m) ||
 768           is_identity_static_mapping (m))
 769         continue;
 770       if (m->external_addr.as_u32 == addr.as_u32)
 771         return 1;
 772   }));
 773   /* *INDENT-ON* */
 774
 775   return 0;
 776 }
 777
 778 static void
 779 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 780                                        ip4_address_t l_addr,
 781                                        u16 l_port,
 782                                        u32 sw_if_index,
 783                                        u16 e_port,
 784                                        u32 vrf_id,
 785                                        snat_protocol_t proto,
 786                                        int addr_only, int is_add, u8 * tag,
 787                                        int twice_nat, int out2in_only,
 788                                        int identity_nat)
 789 {
 790   snat_static_map_resolve_t *rp;
 791
 792   vec_add2 (sm->to_resolve, rp, 1);
 793   rp->l_addr.as_u32 = l_addr.as_u32;
 794   rp->l_port = l_port;
 795   rp->sw_if_index = sw_if_index;
 796   rp->e_port = e_port;
 797   rp->vrf_id = vrf_id;
 798   rp->proto = proto;
 799   rp->addr_only = addr_only;
 800   rp->is_add = is_add;
 801   rp->twice_nat = twice_nat;
 802   rp->out2in_only = out2in_only;
 803   rp->identity_nat = identity_nat;
 804   rp->tag = vec_dup (tag);
 805 }
 806
 807 static u32
 808 get_thread_idx_by_port (u16 e_port)
 809 {
 810   snat_main_t *sm = &snat_main;
 811   u32 thread_idx = sm->num_workers;
 812   if (sm->num_workers > 1)
 813     {
 814       thread_idx =
 815         sm->first_worker_index +
 816         sm->workers[(e_port - 1024) / sm->port_per_thread];
 817     }
 818   return thread_idx;
 819 }
 820
 821 void
 822 snat_static_mapping_del_sessions (snat_main_t * sm,
 823                                   snat_main_per_thread_data_t * tsm,
 824                                   snat_user_key_t u_key, int addr_only,
 825                                   ip4_address_t e_addr, u16 e_port)
 826 {
 827   clib_bihash_kv_8_8_t kv, value;
 828   kv.key = u_key.as_u64;
 829   u64 user_index;
 830   dlist_elt_t *head, *elt;
 831   snat_user_t *u;
 832   snat_session_t *s;
 833   u32 elt_index, head_index, ses_index;
 834   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 835     {
 836       user_index = value.value;
 837       u = pool_elt_at_index (tsm->users, user_index);
 838       if (u->nstaticsessions)
 839         {
 840           head_index = u->sessions_per_user_list_head_index;
 841           head = pool_elt_at_index (tsm->list_pool, head_index);
 842           elt_index = head->next;
 843           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 844           ses_index = elt->value;
 845           while (ses_index != ~0)
 846             {
 847               s = pool_elt_at_index (tsm->sessions, ses_index);
 848               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 849               ses_index = elt->value;
 850
 851               if (!addr_only)
 852                 {
 853                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 854                       (clib_net_to_host_u16 (s->out2in.port) != e_port))
 855                     continue;
 856                 }
 857
 858               if (is_lb_session (s))
 859                 continue;
 860
 861               if (!snat_is_session_static (s))
 862                 continue;
 863
 864               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 865               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 866
 867               if (!addr_only)
 868                 break;
 869             }
 870         }
 871     }
 872 }
 873
 874 void
 875 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 876                                      snat_main_per_thread_data_t * tsm,
 877                                      ip4_address_t l_addr,
 878                                      u16 l_port,
 879                                      u8 protocol,
 880                                      u32 fib_index, int addr_only,
 881                                      ip4_address_t e_addr, u16 e_port)
 882 {
 883   snat_session_t *s;
 884   u32 *indexes_to_free = NULL;
 885   /* *INDENT-OFF* */
 886   pool_foreach (s, tsm->sessions, {
 887     if (s->in2out.fib_index != fib_index ||
 888         s->in2out.addr.as_u32 != l_addr.as_u32)
 889       {
 890         continue;
 891       }
 892     if (!addr_only)
 893       {
 894         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 895             (clib_net_to_host_u16 (s->out2in.port) != e_port) ||
 896             clib_net_to_host_u16 (s->in2out.port) != l_port ||
 897             s->in2out.protocol != protocol)
 898           continue;
 899       }
 900
 901     if (is_lb_session (s))
 902       continue;
 903     if (!snat_is_session_static (s))
 904       continue;
 905     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 906     vec_add1 (indexes_to_free, s - tsm->sessions);
 907     if (!addr_only)
 908       break;
 909   });
 910   /* *INDENT-ON* */
 911   u32 *ses_index;
 912   vec_foreach (ses_index, indexes_to_free)
 913   {
 914     s = pool_elt_at_index (tsm->sessions, *ses_index);
 915     nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
 916   }
 917   vec_free (indexes_to_free);
 918 }
 919
 920 int
 921 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 922                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 923                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 924                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 925                          u8 identity_nat)
 926 {
 927   snat_main_t *sm = &snat_main;
 928   snat_static_mapping_t *m;
 929   snat_session_key_t m_key;
 930   clib_bihash_kv_8_8_t kv, value;
 931   snat_address_t *a = 0;
 932   u32 fib_index = ~0;
 933   snat_interface_t *interface;
 934   int i;
 935   snat_main_per_thread_data_t *tsm;
 936   snat_user_key_t u_key;
 937   snat_user_t *u;
 938   dlist_elt_t *head, *elt;
 939   u32 elt_index, head_index;
 940   u32 ses_index;
 941   u64 user_index;
 942   snat_session_t *s;
 943   snat_static_map_resolve_t *rp, *rp_match = 0;
 944   nat44_lb_addr_port_t *local;
 945   u32 find = ~0;
 946
 947   if (!sm->endpoint_dependent)
 948     {
 949       if (twice_nat || out2in_only)
 950         return VNET_API_ERROR_FEATURE_DISABLED;
 951     }
 952
 953   /* If the external address is a specific interface address */
 954   if (sw_if_index != ~0)
 955     {
 956       ip4_address_t *first_int_addr;
 957
 958       for (i = 0; i < vec_len (sm->to_resolve); i++)
 959         {
 960           rp = sm->to_resolve + i;
 961           if (rp->sw_if_index != sw_if_index ||
 962               rp->l_addr.as_u32 != l_addr.as_u32 ||
 963               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 964             continue;
 965
 966           if (!addr_only)
 967             {
 968               if ((rp->l_port != l_port && rp->e_port != e_port)
 969                   || rp->proto != proto)
 970                 continue;
 971             }
 972
 973           rp_match = rp;
 974           break;
 975         }
 976
 977       /* Might be already set... */
 978       first_int_addr = ip4_interface_first_address
 979         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 980
 981       if (is_add)
 982         {
 983           if (rp_match)
 984             return VNET_API_ERROR_VALUE_EXIST;
 985
 986           snat_add_static_mapping_when_resolved
 987             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 988              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 989
 990           /* DHCP resolution required? */
 991           if (first_int_addr == 0)
 992             {
 993               return 0;
 994             }
 995           else
 996             {
 997               e_addr.as_u32 = first_int_addr->as_u32;
 998               /* Identity mapping? */
 999               if (l_addr.as_u32 == 0)
1000                 l_addr.as_u32 = e_addr.as_u32;
1001             }
1002         }
1003       else
1004         {
1005           if (!rp_match)
1006             return VNET_API_ERROR_NO_SUCH_ENTRY;
1007
1008           vec_del1 (sm->to_resolve, i);
1009
1010           if (first_int_addr)
1011             {
1012               e_addr.as_u32 = first_int_addr->as_u32;
1013               /* Identity mapping? */
1014               if (l_addr.as_u32 == 0)
1015                 l_addr.as_u32 = e_addr.as_u32;
1016             }
1017           else
1018             return 0;
1019         }
1020     }
1021
1022   m_key.addr = e_addr;
1023   m_key.port = addr_only ? 0 : e_port;
1024   m_key.protocol = addr_only ? 0 : proto;
1025   m_key.fib_index = 0;
1026   kv.key = m_key.as_u64;
1027   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1028     m = 0;
1029   else
1030     m = pool_elt_at_index (sm->static_mappings, value.value);
1031
1032   if (is_add)
1033     {
1034       if (m)
1035         {
1036           if (is_identity_static_mapping (m))
1037             {
1038               /* *INDENT-OFF* */
1039               pool_foreach (local, m->locals,
1040               ({
1041                 if (local->vrf_id == vrf_id)
1042                   return VNET_API_ERROR_VALUE_EXIST;
1043               }));
1044               /* *INDENT-ON* */
1045               pool_get (m->locals, local);
1046               local->vrf_id = vrf_id;
1047               local->fib_index =
1048                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1049                                                    nat_fib_src_low);
1050               m_key.addr = m->local_addr;
1051               m_key.port = m->local_port;
1052               m_key.protocol = m->proto;
1053               m_key.fib_index = local->fib_index;
1054               kv.key = m_key.as_u64;
1055               kv.value = m - sm->static_mappings;
1056               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1057               return 0;
1058             }
1059           else
1060             return VNET_API_ERROR_VALUE_EXIST;
1061         }
1062
1063       if (twice_nat && addr_only)
1064         return VNET_API_ERROR_UNSUPPORTED;
1065
1066       /* Convert VRF id to FIB index */
1067       if (vrf_id != ~0)
1068         fib_index =
1069           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1070                                              nat_fib_src_low);
1071       /* If not specified use inside VRF id from SNAT plugin startup config */
1072       else
1073         {
1074           fib_index = sm->inside_fib_index;
1075           vrf_id = sm->inside_vrf_id;
1076           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1077         }
1078
1079       if (!(out2in_only || identity_nat))
1080         {
1081           m_key.addr = l_addr;
1082           m_key.port = addr_only ? 0 : l_port;
1083           m_key.protocol = addr_only ? 0 : proto;
1084           m_key.fib_index = fib_index;
1085           kv.key = m_key.as_u64;
1086           if (!clib_bihash_search_8_8
1087               (&sm->static_mapping_by_local, &kv, &value))
1088             return VNET_API_ERROR_VALUE_EXIST;
1089         }
1090
1091       /* Find external address in allocated addresses and reserve port for
1092          address and port pair mapping when dynamic translations enabled */
1093       if (!(addr_only || sm->static_mapping_only || out2in_only))
1094         {
1095           for (i = 0; i < vec_len (sm->addresses); i++)
1096             {
1097               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1098                 {
1099                   a = sm->addresses + i;
1100                   /* External port must be unused */
1101                   switch (proto)
1102                     {
1103 #define _(N, j, n, s) \
1104                     case SNAT_PROTOCOL_##N: \
1105                       if (a->busy_##n##_port_refcounts[e_port]) \
1106                         return VNET_API_ERROR_INVALID_VALUE; \
1107                       ++a->busy_##n##_port_refcounts[e_port]; \
1108                       if (e_port > 1024) \
1109                         { \
1110                           a->busy_##n##_ports++; \
1111                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1112                         } \
1113                       break;
1114                       foreach_snat_protocol
1115 #undef _
1116                     default:
1117                       nat_elog_info ("unknown protocol");
1118                       return VNET_API_ERROR_INVALID_VALUE_2;
1119                     }
1120                   break;
1121                 }
1122             }
1123           /* External address must be allocated */
1124           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1125             {
1126               if (sw_if_index != ~0)
1127                 {
1128                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1129                     {
1130                       rp = sm->to_resolve + i;
1131                       if (rp->addr_only)
1132                         continue;
1133                       if (rp->sw_if_index != sw_if_index &&
1134                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1135                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1136                           rp->e_port != e_port && rp->proto != proto)
1137                         continue;
1138
1139                       vec_del1 (sm->to_resolve, i);
1140                       break;
1141                     }
1142                 }
1143               return VNET_API_ERROR_NO_SUCH_ENTRY;
1144             }
1145         }
1146
1147       pool_get (sm->static_mappings, m);
1148       clib_memset (m, 0, sizeof (*m));
1149       m->tag = vec_dup (tag);
1150       m->local_addr = l_addr;
1151       m->external_addr = e_addr;
1152       m->twice_nat = twice_nat;
1153       if (out2in_only)
1154         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1155       if (addr_only)
1156         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1157       if (identity_nat)
1158         {
1159           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1160           pool_get (m->locals, local);
1161           local->vrf_id = vrf_id;
1162           local->fib_index = fib_index;
1163         }
1164       else
1165         {
1166           m->vrf_id = vrf_id;
1167           m->fib_index = fib_index;
1168         }
1169       if (!addr_only)
1170         {
1171           m->local_port = l_port;
1172           m->external_port = e_port;
1173           m->proto = proto;
1174         }
1175
1176       if (sm->num_workers > 1)
1177         {
1178           ip4_header_t ip = {
1179             .src_address = m->local_addr,
1180           };
1181           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1182           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1183         }
1184       else
1185         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1186
1187       m_key.addr = m->local_addr;
1188       m_key.port = m->local_port;
1189       m_key.protocol = m->proto;
1190       m_key.fib_index = fib_index;
1191       kv.key = m_key.as_u64;
1192       kv.value = m - sm->static_mappings;
1193       if (!out2in_only)
1194         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1195
1196       m_key.addr = m->external_addr;
1197       m_key.port = m->external_port;
1198       m_key.fib_index = 0;
1199       kv.key = m_key.as_u64;
1200       kv.value = m - sm->static_mappings;
1201       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1202
1203       /* Delete dynamic sessions matching local address (+ local port) */
1204       if (!(sm->static_mapping_only))
1205         {
1206           u_key.addr = m->local_addr;
1207           u_key.fib_index = m->fib_index;
1208           kv.key = u_key.as_u64;
1209           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1210             {
1211               user_index = value.value;
1212               u = pool_elt_at_index (tsm->users, user_index);
1213               if (u->nsessions)
1214                 {
1215                   head_index = u->sessions_per_user_list_head_index;
1216                   head = pool_elt_at_index (tsm->list_pool, head_index);
1217                   elt_index = head->next;
1218                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1219                   ses_index = elt->value;
1220                   while (ses_index != ~0)
1221                     {
1222                       s = pool_elt_at_index (tsm->sessions, ses_index);
1223                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1224                       ses_index = elt->value;
1225
1226                       if (snat_is_session_static (s))
1227                         continue;
1228
1229                       if (!addr_only
1230                           && (clib_net_to_host_u16 (s->in2out.port) !=
1231                               m->local_port))
1232                         continue;
1233
1234                       nat_free_session_data (sm, s,
1235                                              tsm - sm->per_thread_data, 0);
1236                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1237
1238                       if (!addr_only && !sm->endpoint_dependent)
1239                         break;
1240                     }
1241                 }
1242             }
1243         }
1244     }
1245   else
1246     {
1247       if (!m)
1248         {
1249           if (sw_if_index != ~0)
1250             return 0;
1251           else
1252             return VNET_API_ERROR_NO_SUCH_ENTRY;
1253         }
1254
1255       if (identity_nat)
1256         {
1257           if (vrf_id == ~0)
1258             vrf_id = sm->inside_vrf_id;
1259
1260           /* *INDENT-OFF* */
1261           pool_foreach (local, m->locals,
1262           ({
1263             if (local->vrf_id == vrf_id)
1264               find = local - m->locals;
1265           }));
1266           /* *INDENT-ON* */
1267           if (find == ~0)
1268             return VNET_API_ERROR_NO_SUCH_ENTRY;
1269
1270           local = pool_elt_at_index (m->locals, find);
1271           fib_index = local->fib_index;
1272           pool_put (m->locals, local);
1273         }
1274       else
1275         fib_index = m->fib_index;
1276
1277       /* Free external address port */
1278       if (!(addr_only || sm->static_mapping_only || out2in_only))
1279         {
1280           for (i = 0; i < vec_len (sm->addresses); i++)
1281             {
1282               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1283                 {
1284                   a = sm->addresses + i;
1285                   switch (proto)
1286                     {
1287 #define _(N, j, n, s) \
1288                     case SNAT_PROTOCOL_##N: \
1289                       --a->busy_##n##_port_refcounts[e_port]; \
1290                       if (e_port > 1024) \
1291                         { \
1292                           a->busy_##n##_ports--; \
1293                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1294                         } \
1295                       break;
1296                       foreach_snat_protocol
1297 #undef _
1298                     default:
1299                       nat_elog_info ("unknown protocol");
1300                       return VNET_API_ERROR_INVALID_VALUE_2;
1301                     }
1302                   break;
1303                 }
1304             }
1305         }
1306
1307       if (sm->num_workers > 1)
1308         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1309       else
1310         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1311
1312       m_key.addr = m->local_addr;
1313       m_key.port = m->local_port;
1314       m_key.protocol = m->proto;
1315       m_key.fib_index = fib_index;
1316       kv.key = m_key.as_u64;
1317       if (!out2in_only)
1318         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1319
1320       /* Delete session(s) for static mapping if exist */
1321       if (!(sm->static_mapping_only) ||
1322           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1323         {
1324           if (sm->endpoint_dependent)
1325             {
1326               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1327                                                    m->local_port, m->proto,
1328                                                    fib_index, addr_only,
1329                                                    e_addr, e_port);
1330             }
1331           else
1332             {
1333               u_key.addr = m->local_addr;
1334               u_key.fib_index = fib_index;
1335               kv.key = u_key.as_u64;
1336               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1337                                                 e_addr, e_port);
1338             }
1339         }
1340
1341       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1342       if (pool_elts (m->locals))
1343         return 0;
1344
1345       m_key.addr = m->external_addr;
1346       m_key.port = m->external_port;
1347       m_key.fib_index = 0;
1348       kv.key = m_key.as_u64;
1349       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1350
1351       vec_free (m->tag);
1352       vec_free (m->workers);
1353       /* Delete static mapping from pool */
1354       pool_put (sm->static_mappings, m);
1355     }
1356
1357   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1358     return 0;
1359
1360   /* Add/delete external address to FIB */
1361   /* *INDENT-OFF* */
1362   pool_foreach (interface, sm->interfaces,
1363   ({
1364     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1365       continue;
1366
1367     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1368     break;
1369   }));
1370   pool_foreach (interface, sm->output_feature_interfaces,
1371   ({
1372     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1373       continue;
1374
1375     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1376     break;
1377   }));
1378   /* *INDENT-ON* */
1379
1380   return 0;
1381 }
1382
1383 int
1384 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1385                                  snat_protocol_t proto,
1386                                  nat44_lb_addr_port_t * locals, u8 is_add,
1387                                  twice_nat_type_t twice_nat, u8 out2in_only,
1388                                  u8 * tag, u32 affinity)
1389 {
1390   snat_main_t *sm = &snat_main;
1391   snat_static_mapping_t *m;
1392   snat_session_key_t m_key;
1393   clib_bihash_kv_8_8_t kv, value;
1394   snat_address_t *a = 0;
1395   int i;
1396   nat44_lb_addr_port_t *local;
1397   snat_main_per_thread_data_t *tsm;
1398   snat_session_t *s;
1399   uword *bitmap = 0;
1400
1401   if (!sm->endpoint_dependent)
1402     return VNET_API_ERROR_FEATURE_DISABLED;
1403
1404   m_key.addr = e_addr;
1405   m_key.port = e_port;
1406   m_key.protocol = proto;
1407   m_key.fib_index = 0;
1408   kv.key = m_key.as_u64;
1409   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1410     m = 0;
1411   else
1412     m = pool_elt_at_index (sm->static_mappings, value.value);
1413
1414   if (is_add)
1415     {
1416       if (m)
1417         return VNET_API_ERROR_VALUE_EXIST;
1418
1419       if (vec_len (locals) < 2)
1420         return VNET_API_ERROR_INVALID_VALUE;
1421
1422       /* Find external address in allocated addresses and reserve port for
1423          address and port pair mapping when dynamic translations enabled */
1424       if (!(sm->static_mapping_only || out2in_only))
1425         {
1426           for (i = 0; i < vec_len (sm->addresses); i++)
1427             {
1428               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1429                 {
1430                   a = sm->addresses + i;
1431                   /* External port must be unused */
1432                   switch (proto)
1433                     {
1434 #define _(N, j, n, s) \
1435                     case SNAT_PROTOCOL_##N: \
1436                       if (a->busy_##n##_port_refcounts[e_port]) \
1437                         return VNET_API_ERROR_INVALID_VALUE; \
1438                       ++a->busy_##n##_port_refcounts[e_port]; \
1439                       if (e_port > 1024) \
1440                         { \
1441                           a->busy_##n##_ports++; \
1442                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1443                         } \
1444                       break;
1445                       foreach_snat_protocol
1446 #undef _
1447                     default:
1448                       nat_elog_info ("unknown protocol");
1449                       return VNET_API_ERROR_INVALID_VALUE_2;
1450                     }
1451                   break;
1452                 }
1453             }
1454           /* External address must be allocated */
1455           if (!a)
1456             return VNET_API_ERROR_NO_SUCH_ENTRY;
1457         }
1458
1459       pool_get (sm->static_mappings, m);
1460       clib_memset (m, 0, sizeof (*m));
1461       m->tag = vec_dup (tag);
1462       m->external_addr = e_addr;
1463       m->external_port = e_port;
1464       m->proto = proto;
1465       m->twice_nat = twice_nat;
1466       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1467       if (out2in_only)
1468         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1469       m->affinity = affinity;
1470
1471       if (affinity)
1472         m->affinity_per_service_list_head_index =
1473           nat_affinity_get_per_service_list_head_index ();
1474       else
1475         m->affinity_per_service_list_head_index = ~0;
1476
1477       m_key.addr = m->external_addr;
1478       m_key.port = m->external_port;
1479       m_key.protocol = m->proto;
1480       m_key.fib_index = 0;
1481       kv.key = m_key.as_u64;
1482       kv.value = m - sm->static_mappings;
1483       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1484         {
1485           nat_elog_err ("static_mapping_by_external key add failed");
1486           return VNET_API_ERROR_UNSPECIFIED;
1487         }
1488
1489       m_key.fib_index = m->fib_index;
1490       for (i = 0; i < vec_len (locals); i++)
1491         {
1492           locals[i].fib_index =
1493             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1494                                                locals[i].vrf_id,
1495                                                nat_fib_src_low);
1496           m_key.addr = locals[i].addr;
1497           m_key.fib_index = locals[i].fib_index;
1498           if (!out2in_only)
1499             {
1500               m_key.port = locals[i].port;
1501               kv.key = m_key.as_u64;
1502               kv.value = m - sm->static_mappings;
1503               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1504             }
1505           locals[i].prefix = (i == 0) ? locals[i].probability :
1506             (locals[i - 1].prefix + locals[i].probability);
1507           pool_get (m->locals, local);
1508           *local = locals[i];
1509           if (sm->num_workers > 1)
1510             {
1511               ip4_header_t ip = {
1512                 .src_address = locals[i].addr,
1513               };
1514               bitmap =
1515                 clib_bitmap_set (bitmap,
1516                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1517                                  1);
1518             }
1519         }
1520
1521       /* Assign workers */
1522       if (sm->num_workers > 1)
1523         {
1524           /* *INDENT-OFF* */
1525           clib_bitmap_foreach (i, bitmap,
1526             ({
1527                vec_add1(m->workers, i);
1528             }));
1529           /* *INDENT-ON* */
1530         }
1531     }
1532   else
1533     {
1534       if (!m)
1535         return VNET_API_ERROR_NO_SUCH_ENTRY;
1536
1537       if (!is_lb_static_mapping (m))
1538         return VNET_API_ERROR_INVALID_VALUE;
1539
1540       /* Free external address port */
1541       if (!(sm->static_mapping_only || out2in_only))
1542         {
1543           for (i = 0; i < vec_len (sm->addresses); i++)
1544             {
1545               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1546                 {
1547                   a = sm->addresses + i;
1548                   switch (proto)
1549                     {
1550 #define _(N, j, n, s) \
1551                     case SNAT_PROTOCOL_##N: \
1552                       --a->busy_##n##_port_refcounts[e_port]; \
1553                       if (e_port > 1024) \
1554                         { \
1555                           a->busy_##n##_ports--; \
1556                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1557                         } \
1558                       break;
1559                       foreach_snat_protocol
1560 #undef _
1561                     default:
1562                       nat_elog_info ("unknown protocol");
1563                       return VNET_API_ERROR_INVALID_VALUE_2;
1564                     }
1565                   break;
1566                 }
1567             }
1568         }
1569
1570       m_key.addr = m->external_addr;
1571       m_key.port = m->external_port;
1572       m_key.protocol = m->proto;
1573       m_key.fib_index = 0;
1574       kv.key = m_key.as_u64;
1575       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1576         {
1577           nat_elog_err ("static_mapping_by_external key del failed");
1578           return VNET_API_ERROR_UNSPECIFIED;
1579         }
1580
1581       /* *INDENT-OFF* */
1582       pool_foreach (local, m->locals,
1583       ({
1584           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1585                             nat_fib_src_low);
1586           m_key.addr = local->addr;
1587           if (!out2in_only)
1588             {
1589               m_key.port = local->port;
1590               m_key.fib_index = local->fib_index;
1591               kv.key = m_key.as_u64;
1592               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1593                 {
1594                   nat_elog_err ("static_mapping_by_local key del failed");
1595                   return VNET_API_ERROR_UNSPECIFIED;
1596                 }
1597             }
1598
1599           if (sm->num_workers > 1)
1600             {
1601               ip4_header_t ip = {
1602                 .src_address = local->addr,
1603               };
1604               tsm = vec_elt_at_index (sm->per_thread_data,
1605                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1606             }
1607           else
1608             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1609
1610           /* Delete sessions */
1611           pool_foreach (s, tsm->sessions, {
1612             if (!(is_lb_session (s)))
1613               continue;
1614
1615             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1616                 (clib_net_to_host_u16 (s->in2out.port) != local->port))
1617               continue;
1618
1619             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1620             nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
1621           });
1622       }));
1623       /* *INDENT-ON* */
1624       if (m->affinity)
1625         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1626       pool_free (m->locals);
1627       vec_free (m->tag);
1628       vec_free (m->workers);
1629
1630       pool_put (sm->static_mappings, m);
1631     }
1632
1633   return 0;
1634 }
1635
1636 int
1637 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1638                                        ip4_address_t l_addr, u16 l_port,
1639                                        snat_protocol_t proto, u32 vrf_id,
1640                                        u8 probability, u8 is_add)
1641 {
1642   snat_main_t *sm = &snat_main;
1643   snat_static_mapping_t *m = 0;
1644   snat_session_key_t m_key;
1645   clib_bihash_kv_8_8_t kv, value;
1646   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1647   snat_main_per_thread_data_t *tsm;
1648   snat_session_t *s;
1649   u32 *locals = 0;
1650   uword *bitmap = 0;
1651   int i;
1652
1653   if (!sm->endpoint_dependent)
1654     return VNET_API_ERROR_FEATURE_DISABLED;
1655
1656   m_key.addr = e_addr;
1657   m_key.port = e_port;
1658   m_key.protocol = proto;
1659   m_key.fib_index = 0;
1660   kv.key = m_key.as_u64;
1661   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1662     m = pool_elt_at_index (sm->static_mappings, value.value);
1663
1664   if (!m)
1665     return VNET_API_ERROR_NO_SUCH_ENTRY;
1666
1667   if (!is_lb_static_mapping (m))
1668     return VNET_API_ERROR_INVALID_VALUE;
1669
1670   /* *INDENT-OFF* */
1671   pool_foreach (local, m->locals,
1672   ({
1673     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1674         (local->vrf_id == vrf_id))
1675       {
1676         match_local = local;
1677         break;
1678       }
1679   }));
1680   /* *INDENT-ON* */
1681
1682   if (is_add)
1683     {
1684       if (match_local)
1685         return VNET_API_ERROR_VALUE_EXIST;
1686
1687       pool_get (m->locals, local);
1688       clib_memset (local, 0, sizeof (*local));
1689       local->addr.as_u32 = l_addr.as_u32;
1690       local->port = l_port;
1691       local->probability = probability;
1692       local->vrf_id = vrf_id;
1693       local->fib_index =
1694         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1695                                            nat_fib_src_low);
1696
1697       if (!is_out2in_only_static_mapping (m))
1698         {
1699           m_key.addr = l_addr;
1700           m_key.port = l_port;
1701           m_key.fib_index = local->fib_index;
1702           kv.key = m_key.as_u64;
1703           kv.value = m - sm->static_mappings;
1704           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1705             nat_elog_err ("static_mapping_by_local key add failed");
1706         }
1707     }
1708   else
1709     {
1710       if (!match_local)
1711         return VNET_API_ERROR_NO_SUCH_ENTRY;
1712
1713       if (pool_elts (m->locals) < 3)
1714         return VNET_API_ERROR_UNSPECIFIED;
1715
1716       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1717                         nat_fib_src_low);
1718
1719       if (!is_out2in_only_static_mapping (m))
1720         {
1721           m_key.addr = l_addr;
1722           m_key.port = l_port;
1723           m_key.fib_index = match_local->fib_index;
1724           kv.key = m_key.as_u64;
1725           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1726             nat_elog_err ("static_mapping_by_local key del failed");
1727         }
1728
1729       if (sm->num_workers > 1)
1730         {
1731           ip4_header_t ip = {
1732             .src_address = local->addr,
1733           };
1734           tsm = vec_elt_at_index (sm->per_thread_data,
1735                                   sm->worker_in2out_cb (&ip, m->fib_index,
1736                                                         0));
1737         }
1738       else
1739         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1740
1741       /* Delete sessions */
1742       /* *INDENT-OFF* */
1743       pool_foreach (s, tsm->sessions, {
1744         if (!(is_lb_session (s)))
1745           continue;
1746
1747         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1748             (clib_net_to_host_u16 (s->in2out.port) != match_local->port))
1749           continue;
1750
1751         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1752         nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
1753       });
1754       /* *INDENT-ON* */
1755
1756       pool_put (m->locals, match_local);
1757     }
1758
1759   vec_free (m->workers);
1760
1761   /* *INDENT-OFF* */
1762   pool_foreach (local, m->locals,
1763   ({
1764     vec_add1 (locals, local - m->locals);
1765     if (sm->num_workers > 1)
1766       {
1767         ip4_header_t ip;
1768         ip.src_address.as_u32 = local->addr.as_u32,
1769         bitmap = clib_bitmap_set (bitmap,
1770                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1771                                   1);
1772       }
1773   }));
1774   /* *INDENT-ON* */
1775
1776   ASSERT (vec_len (locals) > 1);
1777
1778   local = pool_elt_at_index (m->locals, locals[0]);
1779   local->prefix = local->probability;
1780   for (i = 1; i < vec_len (locals); i++)
1781     {
1782       local = pool_elt_at_index (m->locals, locals[i]);
1783       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1784       local->prefix = local->probability + prev_local->prefix;
1785     }
1786
1787   /* Assign workers */
1788   if (sm->num_workers > 1)
1789     {
1790       /* *INDENT-OFF* */
1791       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1792       /* *INDENT-ON* */
1793     }
1794
1795   return 0;
1796 }
1797
1798 int
1799 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1800                   u8 twice_nat)
1801 {
1802   snat_address_t *a = 0;
1803   snat_session_t *ses;
1804   u32 *ses_to_be_removed = 0, *ses_index;
1805   snat_main_per_thread_data_t *tsm;
1806   snat_static_mapping_t *m;
1807   snat_interface_t *interface;
1808   int i;
1809   snat_address_t *addresses =
1810     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1811
1812   /* Find SNAT address */
1813   for (i = 0; i < vec_len (addresses); i++)
1814     {
1815       if (addresses[i].addr.as_u32 == addr.as_u32)
1816         {
1817           a = addresses + i;
1818           break;
1819         }
1820     }
1821   if (!a)
1822     return VNET_API_ERROR_NO_SUCH_ENTRY;
1823
1824   if (delete_sm)
1825     {
1826       /* *INDENT-OFF* */
1827       pool_foreach (m, sm->static_mappings,
1828       ({
1829           if (m->external_addr.as_u32 == addr.as_u32)
1830             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1831                                             m->local_port, m->external_port,
1832                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1833                                             m->proto, 0, m->twice_nat,
1834                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1835       }));
1836       /* *INDENT-ON* */
1837     }
1838   else
1839     {
1840       /* Check if address is used in some static mapping */
1841       if (is_snat_address_used_in_static_mapping (sm, addr))
1842         {
1843           nat_elog_notice ("address used in static mapping");
1844           return VNET_API_ERROR_UNSPECIFIED;
1845         }
1846     }
1847
1848   if (a->fib_index != ~0)
1849     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1850
1851   /* Delete sessions using address */
1852   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1853     {
1854       /* *INDENT-OFF* */
1855       vec_foreach (tsm, sm->per_thread_data)
1856         {
1857           pool_foreach (ses, tsm->sessions, ({
1858             if (ses->out2in.addr.as_u32 == addr.as_u32)
1859               {
1860                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1861                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1862               }
1863           }));
1864
1865           if (sm->endpoint_dependent){
1866               vec_foreach (ses_index, ses_to_be_removed)
1867                 {
1868                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1869                   nat44_ed_delete_session (sm, ses, tsm - sm->per_thread_data, 1);
1870                 }
1871           }else{
1872               vec_foreach (ses_index, ses_to_be_removed)
1873                 {
1874                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1875                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1876                 }
1877           }
1878
1879           vec_free (ses_to_be_removed);
1880         }
1881       /* *INDENT-ON* */
1882     }
1883
1884 #define _(N, i, n, s) \
1885   vec_free (a->busy_##n##_ports_per_thread);
1886   foreach_snat_protocol
1887 #undef _
1888     if (twice_nat)
1889     {
1890       vec_del1 (sm->twice_nat_addresses, i);
1891       return 0;
1892     }
1893   else
1894     vec_del1 (sm->addresses, i);
1895
1896   /* Delete external address from FIB */
1897   /* *INDENT-OFF* */
1898   pool_foreach (interface, sm->interfaces,
1899   ({
1900     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1901       continue;
1902
1903     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1904     break;
1905   }));
1906   pool_foreach (interface, sm->output_feature_interfaces,
1907   ({
1908     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1909       continue;
1910
1911     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1912     break;
1913   }));
1914   /* *INDENT-ON* */
1915
1916   return 0;
1917 }
1918
1919 int
1920 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1921 {
1922   snat_main_t *sm = &snat_main;
1923   snat_interface_t *i;
1924   const char *feature_name, *del_feature_name;
1925   snat_address_t *ap;
1926   snat_static_mapping_t *m;
1927   snat_det_map_t *dm;
1928   nat_outside_fib_t *outside_fib;
1929   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1930                                                        sw_if_index);
1931
1932   if (sm->out2in_dpo && !is_inside)
1933     return VNET_API_ERROR_UNSUPPORTED;
1934
1935   /* *INDENT-OFF* */
1936   pool_foreach (i, sm->output_feature_interfaces,
1937   ({
1938     if (i->sw_if_index == sw_if_index)
1939       return VNET_API_ERROR_VALUE_EXIST;
1940   }));
1941   /* *INDENT-ON* */
1942
1943   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1944     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1945   else
1946     {
1947       if (sm->num_workers > 1 && !sm->deterministic)
1948         feature_name =
1949           is_inside ? "nat44-in2out-worker-handoff" :
1950           "nat44-out2in-worker-handoff";
1951       else if (sm->deterministic)
1952         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1953       else if (sm->endpoint_dependent)
1954         {
1955           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1956         }
1957       else
1958         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1959     }
1960
1961   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1962     sm->fq_in2out_index =
1963       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1964
1965   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1966     sm->fq_out2in_index =
1967       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1968
1969   if (!is_inside)
1970     {
1971       /* *INDENT-OFF* */
1972       vec_foreach (outside_fib, sm->outside_fibs)
1973         {
1974           if (outside_fib->fib_index == fib_index)
1975             {
1976               if (is_del)
1977                 {
1978                   outside_fib->refcount--;
1979                   if (!outside_fib->refcount)
1980                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1981                 }
1982               else
1983                 outside_fib->refcount++;
1984               goto feature_set;
1985             }
1986         }
1987       /* *INDENT-ON* */
1988       if (!is_del)
1989         {
1990           vec_add2 (sm->outside_fibs, outside_fib, 1);
1991           outside_fib->refcount = 1;
1992           outside_fib->fib_index = fib_index;
1993         }
1994     }
1995 feature_set:
1996   /* *INDENT-OFF* */
1997   pool_foreach (i, sm->interfaces,
1998   ({
1999     if (i->sw_if_index == sw_if_index)
2000       {
2001         if (is_del)
2002           {
2003             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
2004               {
2005                 if (is_inside)
2006                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
2007                 else
2008                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
2009
2010                 if (sm->num_workers > 1 && !sm->deterministic)
2011                   {
2012                     del_feature_name = "nat44-handoff-classify";
2013                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2014                                                  "nat44-out2in-worker-handoff";
2015                   }
2016                 else if (sm->deterministic)
2017                   {
2018                     del_feature_name = "nat44-det-classify";
2019                     feature_name = !is_inside ?  "nat44-det-in2out" :
2020                                                  "nat44-det-out2in";
2021                   }
2022                 else if (sm->endpoint_dependent)
2023                   {
2024                     del_feature_name = "nat44-ed-classify";
2025                     feature_name = !is_inside ?  "nat-pre-in2out" :
2026                                                  "nat-pre-out2in";
2027                   }
2028                 else
2029                   {
2030                     del_feature_name = "nat44-classify";
2031                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2032                   }
2033
2034                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2035                 if (rv)
2036                   return rv;
2037                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2038                                              sw_if_index, 0, 0, 0);
2039                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2040                                              sw_if_index, 1, 0, 0);
2041                 if (!is_inside)
2042                   {
2043                     if (sm->endpoint_dependent)
2044                       vnet_feature_enable_disable ("ip4-local",
2045                                                    "nat44-ed-hairpinning",
2046                                                    sw_if_index, 1, 0, 0);
2047                     else if (!sm->deterministic)
2048                       vnet_feature_enable_disable ("ip4-local",
2049                                                    "nat44-hairpinning",
2050                                                    sw_if_index, 1, 0, 0);
2051                   }
2052               }
2053             else
2054               {
2055                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2056                 if (rv)
2057                   return rv;
2058                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2059                                              sw_if_index, 0, 0, 0);
2060                 pool_put (sm->interfaces, i);
2061                 if (is_inside)
2062                   {
2063                     if (sm->endpoint_dependent)
2064                       vnet_feature_enable_disable ("ip4-local",
2065                                                    "nat44-ed-hairpinning",
2066                                                    sw_if_index, 0, 0, 0);
2067                     else if (!sm->deterministic)
2068                       vnet_feature_enable_disable ("ip4-local",
2069                                                    "nat44-hairpinning",
2070                                                    sw_if_index, 0, 0, 0);
2071                   }
2072               }
2073           }
2074         else
2075           {
2076             if ((nat_interface_is_inside(i) && is_inside) ||
2077                 (nat_interface_is_outside(i) && !is_inside))
2078               return 0;
2079
2080             if (sm->num_workers > 1 && !sm->deterministic)
2081               {
2082                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2083                                                  "nat44-out2in-worker-handoff";
2084                 feature_name = "nat44-handoff-classify";
2085               }
2086             else if (sm->deterministic)
2087               {
2088                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
2089                                                  "nat44-det-out2in";
2090                 feature_name = "nat44-det-classify";
2091               }
2092             else if (sm->endpoint_dependent)
2093               {
2094                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2095                                                  "nat-pre-out2in";
2096
2097                 feature_name = "nat44-ed-classify";
2098               }
2099             else
2100               {
2101                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2102                 feature_name = "nat44-classify";
2103               }
2104
2105             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2106             if (rv)
2107               return rv;
2108             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2109                                          sw_if_index, 0, 0, 0);
2110             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2111                                          sw_if_index, 1, 0, 0);
2112             if (!is_inside)
2113               {
2114                 if (sm->endpoint_dependent)
2115                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2116                                                sw_if_index, 0, 0, 0);
2117                 else if (!sm->deterministic)
2118                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2119                                                sw_if_index, 0, 0, 0);
2120               }
2121             goto set_flags;
2122           }
2123
2124         goto fib;
2125       }
2126   }));
2127   /* *INDENT-ON* */
2128
2129   if (is_del)
2130     return VNET_API_ERROR_NO_SUCH_ENTRY;
2131
2132   pool_get (sm->interfaces, i);
2133   i->sw_if_index = sw_if_index;
2134   i->flags = 0;
2135   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2136                                0);
2137
2138   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2139   if (rv)
2140     return rv;
2141
2142   if (is_inside && !sm->out2in_dpo)
2143     {
2144       if (sm->endpoint_dependent)
2145         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2146                                      sw_if_index, 1, 0, 0);
2147       else if (!sm->deterministic)
2148         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2149                                      sw_if_index, 1, 0, 0);
2150     }
2151
2152 set_flags:
2153   if (is_inside)
2154     {
2155       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2156       return 0;
2157     }
2158   else
2159     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2160
2161   /* Add/delete external addresses to FIB */
2162 fib:
2163   /* *INDENT-OFF* */
2164   vec_foreach (ap, sm->addresses)
2165     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2166
2167   pool_foreach (m, sm->static_mappings,
2168   ({
2169     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2170       continue;
2171
2172     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2173   }));
2174
2175   pool_foreach (dm, sm->det_maps,
2176   ({
2177     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2178   }));
2179   /* *INDENT-ON* */
2180
2181   return 0;
2182 }
2183
2184 int
2185 snat_interface_add_del_output_feature (u32 sw_if_index,
2186                                        u8 is_inside, int is_del)
2187 {
2188   snat_main_t *sm = &snat_main;
2189   snat_interface_t *i;
2190   snat_address_t *ap;
2191   snat_static_mapping_t *m;
2192   nat_outside_fib_t *outside_fib;
2193   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2194                                                        sw_if_index);
2195
2196
2197   if (sm->deterministic ||
2198       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2199     return VNET_API_ERROR_UNSUPPORTED;
2200
2201   /* *INDENT-OFF* */
2202   pool_foreach (i, sm->interfaces,
2203   ({
2204     if (i->sw_if_index == sw_if_index)
2205       return VNET_API_ERROR_VALUE_EXIST;
2206   }));
2207   /* *INDENT-ON* */
2208
2209   if (!is_inside)
2210     {
2211       /* *INDENT-OFF* */
2212       vec_foreach (outside_fib, sm->outside_fibs)
2213         {
2214           if (outside_fib->fib_index == fib_index)
2215             {
2216               if (is_del)
2217                 {
2218                   outside_fib->refcount--;
2219                   if (!outside_fib->refcount)
2220                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2221                 }
2222               else
2223                 outside_fib->refcount++;
2224               goto feature_set;
2225             }
2226         }
2227       /* *INDENT-ON* */
2228       if (!is_del)
2229         {
2230           vec_add2 (sm->outside_fibs, outside_fib, 1);
2231           outside_fib->refcount = 1;
2232           outside_fib->fib_index = fib_index;
2233         }
2234     }
2235
2236 feature_set:
2237   if (is_inside)
2238     {
2239       if (sm->endpoint_dependent)
2240         {
2241           int rv =
2242             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2243           if (rv)
2244             return rv;
2245           rv =
2246             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2247                                                             !is_del);
2248           if (rv)
2249             return rv;
2250           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2251                                        sw_if_index, !is_del, 0, 0);
2252           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2253                                        sw_if_index, !is_del, 0, 0);
2254         }
2255       else
2256         {
2257           int rv =
2258             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2259           if (rv)
2260             return rv;
2261           rv =
2262             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2263                                                             !is_del);
2264           if (rv)
2265             return rv;
2266           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2267                                        sw_if_index, !is_del, 0, 0);
2268           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2269                                        sw_if_index, !is_del, 0, 0);
2270         }
2271       goto fq;
2272     }
2273
2274   if (sm->num_workers > 1)
2275     {
2276       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2277       if (rv)
2278         return rv;
2279       rv =
2280         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2281       if (rv)
2282         return rv;
2283       vnet_feature_enable_disable ("ip4-unicast",
2284                                    "nat44-out2in-worker-handoff",
2285                                    sw_if_index, !is_del, 0, 0);
2286       vnet_feature_enable_disable ("ip4-output",
2287                                    "nat44-in2out-output-worker-handoff",
2288                                    sw_if_index, !is_del, 0, 0);
2289     }
2290   else
2291     {
2292       if (sm->endpoint_dependent)
2293         {
2294           int rv =
2295             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2296           if (rv)
2297             return rv;
2298           rv =
2299             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2300                                                             !is_del);
2301           if (rv)
2302             return rv;
2303           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2304                                        sw_if_index, !is_del, 0, 0);
2305           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2306                                        sw_if_index, !is_del, 0, 0);
2307         }
2308       else
2309         {
2310           int rv =
2311             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2312           if (rv)
2313             return rv;
2314           rv =
2315             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2316                                                             !is_del);
2317           if (rv)
2318             return rv;
2319           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2320                                        sw_if_index, !is_del, 0, 0);
2321           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2322                                        sw_if_index, !is_del, 0, 0);
2323         }
2324     }
2325
2326 fq:
2327   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2328     sm->fq_in2out_output_index =
2329       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2330
2331   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2332     sm->fq_out2in_index =
2333       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2334
2335   /* *INDENT-OFF* */
2336   pool_foreach (i, sm->output_feature_interfaces,
2337   ({
2338     if (i->sw_if_index == sw_if_index)
2339       {
2340         if (is_del)
2341           pool_put (sm->output_feature_interfaces, i);
2342         else
2343           return VNET_API_ERROR_VALUE_EXIST;
2344
2345         goto fib;
2346       }
2347   }));
2348   /* *INDENT-ON* */
2349
2350   if (is_del)
2351     return VNET_API_ERROR_NO_SUCH_ENTRY;
2352
2353   pool_get (sm->output_feature_interfaces, i);
2354   i->sw_if_index = sw_if_index;
2355   i->flags = 0;
2356   if (is_inside)
2357     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2358   else
2359     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2360
2361   /* Add/delete external addresses to FIB */
2362 fib:
2363   if (is_inside)
2364     return 0;
2365
2366   /* *INDENT-OFF* */
2367   vec_foreach (ap, sm->addresses)
2368     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2369
2370   pool_foreach (m, sm->static_mappings,
2371   ({
2372     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2373       continue;
2374
2375     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2376   }));
2377   /* *INDENT-ON* */
2378
2379   return 0;
2380 }
2381
2382 int
2383 snat_set_workers (uword * bitmap)
2384 {
2385   snat_main_t *sm = &snat_main;
2386   int i, j = 0;
2387
2388   if (sm->num_workers < 2)
2389     return VNET_API_ERROR_FEATURE_DISABLED;
2390
2391   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2392     return VNET_API_ERROR_INVALID_WORKER;
2393
2394   vec_free (sm->workers);
2395   /* *INDENT-OFF* */
2396   clib_bitmap_foreach (i, bitmap,
2397     ({
2398       vec_add1(sm->workers, i);
2399       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2400       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2401       j++;
2402     }));
2403   /* *INDENT-ON* */
2404
2405   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2406   sm->num_snat_thread = _vec_len (sm->workers);
2407
2408   return 0;
2409 }
2410
2411 static void
2412 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2413                          u32 old_fib_index)
2414 {
2415   snat_main_t *sm = &snat_main;
2416   nat_outside_fib_t *outside_fib;
2417   snat_interface_t *i;
2418   u8 is_add = 1;
2419   u8 match = 0;
2420
2421   if (new_fib_index == old_fib_index)
2422     return;
2423
2424   if (!vec_len (sm->outside_fibs))
2425     return;
2426
2427   /* *INDENT-OFF* */
2428   pool_foreach (i, sm->interfaces,
2429     ({
2430       if (i->sw_if_index == sw_if_index)
2431         {
2432           if (!(nat_interface_is_outside (i)))
2433             return;
2434           match = 1;
2435         }
2436     }));
2437
2438   pool_foreach (i, sm->output_feature_interfaces,
2439     ({
2440       if (i->sw_if_index == sw_if_index)
2441         {
2442           if (!(nat_interface_is_outside (i)))
2443             return;
2444           match = 1;
2445         }
2446     }));
2447   /* *INDENT-ON* */
2448
2449   if (!match)
2450     return;
2451
2452   vec_foreach (outside_fib, sm->outside_fibs)
2453   {
2454     if (outside_fib->fib_index == old_fib_index)
2455       {
2456         outside_fib->refcount--;
2457         if (!outside_fib->refcount)
2458           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2459         break;
2460       }
2461   }
2462
2463   vec_foreach (outside_fib, sm->outside_fibs)
2464   {
2465     if (outside_fib->fib_index == new_fib_index)
2466       {
2467         outside_fib->refcount++;
2468         is_add = 0;
2469         break;
2470       }
2471   }
2472
2473   if (is_add)
2474     {
2475       vec_add2 (sm->outside_fibs, outside_fib, 1);
2476       outside_fib->refcount = 1;
2477       outside_fib->fib_index = new_fib_index;
2478     }
2479 }
2480
2481 static void
2482 snat_ip4_table_bind (ip4_main_t * im,
2483                      uword opaque,
2484                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2485 {
2486   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2487 }
2488
2489 static void
2490 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2491                                        uword opaque,
2492                                        u32 sw_if_index,
2493                                        ip4_address_t * address,
2494                                        u32 address_length,
2495                                        u32 if_address_index, u32 is_delete);
2496
2497 static void
2498 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2499                                  uword opaque,
2500                                  u32 sw_if_index,
2501                                  ip4_address_t * address,
2502                                  u32 address_length,
2503                                  u32 if_address_index, u32 is_delete);
2504
2505 static int
2506 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2507                                  u32 fib_index,
2508                                  u32 thread_index,
2509                                  snat_session_key_t * k,
2510                                  u16 port_per_thread, u32 snat_thread_index);
2511
2512 void
2513 test_ed_make_split ()
2514 {
2515   ip4_address_t l_addr;
2516   l_addr.as_u8[0] = 1;
2517   l_addr.as_u8[1] = 1;
2518   l_addr.as_u8[2] = 1;
2519   l_addr.as_u8[3] = 1;
2520   ip4_address_t r_addr;
2521   r_addr.as_u8[0] = 2;
2522   r_addr.as_u8[1] = 2;
2523   r_addr.as_u8[2] = 2;
2524   r_addr.as_u8[3] = 2;
2525   u16 l_port = 40001;
2526   u16 r_port = 40301;
2527   u8 proto = 9;
2528   u32 fib_index = 9000001;
2529   u64 value = ~0ULL;
2530   clib_bihash_kv_16_8_t kv;
2531   make_ed_kv (&l_addr, &r_addr, proto, fib_index, l_port, r_port, value, &kv);
2532   ip4_address_t l_addr2;
2533   ip4_address_t r_addr2;
2534   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2535   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2536   u16 l_port2 = 0;
2537   u16 r_port2 = 0;
2538   u8 proto2 = 0;
2539   u32 fib_index2 = 0;
2540   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2541                &r_port2);
2542   u64 value2 = kv.value;
2543   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2544   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2545   ASSERT (l_port == l_port2);
2546   ASSERT (r_port == r_port2);
2547   ASSERT (proto == proto2);
2548   ASSERT (fib_index == fib_index2);
2549   ASSERT (value == value2);
2550 }
2551
2552 static clib_error_t *
2553 snat_init (vlib_main_t * vm)
2554 {
2555   snat_main_t *sm = &snat_main;
2556   clib_error_t *error = 0;
2557   ip4_main_t *im = &ip4_main;
2558   ip_lookup_main_t *lm = &im->lookup_main;
2559   uword *p;
2560   vlib_thread_registration_t *tr;
2561   vlib_thread_main_t *tm = vlib_get_thread_main ();
2562   uword *bitmap = 0;
2563   u32 i;
2564   ip4_add_del_interface_address_callback_t cb4;
2565   vlib_node_t *node;
2566
2567   sm->vnet_main = vnet_get_main ();
2568   sm->ip4_main = im;
2569   sm->ip4_lookup_main = lm;
2570   sm->api_main = vlibapi_get_main ();
2571   sm->first_worker_index = 0;
2572   sm->num_workers = 0;
2573   sm->num_snat_thread = 1;
2574   sm->workers = 0;
2575   sm->port_per_thread = 0xffff - 1024;
2576   sm->fq_in2out_index = ~0;
2577   sm->fq_in2out_output_index = ~0;
2578   sm->fq_out2in_index = ~0;
2579
2580   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2581   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2582   sm->forwarding_enabled = 0;
2583   sm->log_class = vlib_log_register_class ("nat", 0);
2584   sm->log_level = SNAT_LOG_ERROR;
2585   sm->mss_clamping = 0;
2586
2587   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2588   sm->error_node_index = node->index;
2589
2590   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2591   sm->pre_in2out_node_index = node->index;
2592   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2593   sm->pre_out2in_node_index = node->index;
2594
2595   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2596   sm->pre_in2out_node_index = node->index;
2597
2598   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2599   sm->pre_out2in_node_index = node->index;
2600
2601   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2602   sm->in2out_node_index = node->index;
2603   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2604   sm->in2out_output_node_index = node->index;
2605   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2606   sm->in2out_fast_node_index = node->index;
2607   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2608   sm->in2out_slowpath_node_index = node->index;
2609   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2610   sm->in2out_slowpath_output_node_index = node->index;
2611
2612   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2613   sm->ed_in2out_node_index = node->index;
2614   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2615   sm->ed_in2out_slowpath_node_index = node->index;
2616
2617   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2618   sm->out2in_node_index = node->index;
2619   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2620   sm->out2in_fast_node_index = node->index;
2621
2622   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2623   sm->ed_out2in_node_index = node->index;
2624   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2625   sm->ed_out2in_slowpath_node_index = node->index;
2626
2627   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2628   sm->det_in2out_node_index = node->index;
2629   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2630   sm->det_out2in_node_index = node->index;
2631
2632   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2633   sm->hairpinning_node_index = node->index;
2634   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2635   sm->hairpin_dst_node_index = node->index;
2636   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2637   sm->hairpin_src_node_index = node->index;
2638   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2639   sm->ed_hairpinning_node_index = node->index;
2640   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2641   sm->ed_hairpin_dst_node_index = node->index;
2642   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2643   sm->ed_hairpin_src_node_index = node->index;
2644
2645   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2646   if (p)
2647     {
2648       tr = (vlib_thread_registration_t *) p[0];
2649       if (tr)
2650         {
2651           sm->num_workers = tr->count;
2652           sm->first_worker_index = tr->first_index;
2653         }
2654     }
2655
2656   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2657
2658   /* Use all available workers by default */
2659   if (sm->num_workers > 1)
2660     {
2661       for (i = 0; i < sm->num_workers; i++)
2662         bitmap = clib_bitmap_set (bitmap, i, 1);
2663       // sets thread indexes for workes
2664       snat_set_workers (bitmap);
2665       clib_bitmap_free (bitmap);
2666     }
2667   else
2668     {
2669       sm->per_thread_data[0].snat_thread_index = 0;
2670     }
2671
2672   error = snat_api_init (vm, sm);
2673   if (error)
2674     return error;
2675
2676   /* Set up the interface address add/del callback */
2677   cb4.function = snat_ip4_add_del_interface_address_cb;
2678   cb4.function_opaque = 0;
2679
2680   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2681
2682   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2683   cb4.function_opaque = 0;
2684
2685   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2686
2687   nat_dpo_module_init ();
2688
2689   /* Init counters */
2690   sm->total_users.name = "total-users";
2691   sm->total_users.stat_segment_name = "/nat44/total-users";
2692   vlib_validate_simple_counter (&sm->total_users, 0);
2693   vlib_zero_simple_counter (&sm->total_users, 0);
2694   sm->total_sessions.name = "total-sessions";
2695   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2696   vlib_validate_simple_counter (&sm->total_sessions, 0);
2697   vlib_zero_simple_counter (&sm->total_sessions, 0);
2698
2699   /* Init IPFIX logging */
2700   snat_ipfix_logging_init (vm);
2701
2702   /* Init NAT64 */
2703   error = nat64_init (vm);
2704   if (error)
2705     return error;
2706
2707   nat66_init (vm);
2708
2709   ip4_table_bind_callback_t cbt4 = {
2710     .function = snat_ip4_table_bind,
2711   };
2712   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2713
2714   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2715                                         FIB_SOURCE_PRIORITY_HI,
2716                                         FIB_SOURCE_BH_SIMPLE);
2717   nat_fib_src_low = fib_source_allocate ("nat-low",
2718                                          FIB_SOURCE_PRIORITY_LOW,
2719                                          FIB_SOURCE_BH_SIMPLE);
2720
2721   test_ed_make_split ();
2722   return error;
2723 }
2724
2725 VLIB_INIT_FUNCTION (snat_init);
2726
2727 void
2728 snat_free_outside_address_and_port (snat_address_t * addresses,
2729                                     u32 thread_index, snat_session_key_t * k)
2730 {
2731   snat_address_t *a;
2732   u32 address_index;
2733   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2734
2735   for (address_index = 0; address_index < vec_len (addresses);
2736        address_index++)
2737     {
2738       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2739         break;
2740     }
2741
2742   ASSERT (address_index < vec_len (addresses));
2743
2744   a = addresses + address_index;
2745
2746   switch (k->protocol)
2747     {
2748 #define _(N, i, n, s) \
2749     case SNAT_PROTOCOL_##N: \
2750       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2751       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2752       a->busy_##n##_ports--; \
2753       a->busy_##n##_ports_per_thread[thread_index]--; \
2754       break;
2755       foreach_snat_protocol
2756 #undef _
2757     default:
2758       nat_elog_info ("unknown protocol");
2759       return;
2760     }
2761 }
2762
2763 static int
2764 nat_set_outside_address_and_port (snat_address_t * addresses,
2765                                   u32 thread_index, snat_session_key_t * k)
2766 {
2767   snat_address_t *a = 0;
2768   u32 address_index;
2769   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2770
2771   for (address_index = 0; address_index < vec_len (addresses);
2772        address_index++)
2773     {
2774       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2775         continue;
2776
2777       a = addresses + address_index;
2778       switch (k->protocol)
2779         {
2780 #define _(N, j, n, s) \
2781         case SNAT_PROTOCOL_##N: \
2782           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2783             return VNET_API_ERROR_INSTANCE_IN_USE; \
2784           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2785           a->busy_##n##_ports_per_thread[thread_index]++; \
2786           a->busy_##n##_ports++; \
2787           return 0;
2788           foreach_snat_protocol
2789 #undef _
2790         default:
2791           nat_elog_info ("unknown protocol");
2792           return 1;
2793         }
2794     }
2795
2796   return VNET_API_ERROR_NO_SUCH_ENTRY;
2797 }
2798
2799 int
2800 snat_static_mapping_match (snat_main_t * sm,
2801                            snat_session_key_t match,
2802                            snat_session_key_t * mapping,
2803                            u8 by_external,
2804                            u8 * is_addr_only,
2805                            twice_nat_type_t * twice_nat,
2806                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2807                            u8 * is_identity_nat)
2808 {
2809   clib_bihash_kv_8_8_t kv, value;
2810   snat_static_mapping_t *m;
2811   snat_session_key_t m_key;
2812   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2813   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2814   u8 backend_index;
2815   nat44_lb_addr_port_t *local;
2816
2817   m_key.fib_index = match.fib_index;
2818   if (by_external)
2819     {
2820       mapping_hash = &sm->static_mapping_by_external;
2821       m_key.fib_index = 0;
2822     }
2823
2824   m_key.addr = match.addr;
2825   m_key.port = clib_net_to_host_u16 (match.port);
2826   m_key.protocol = match.protocol;
2827
2828   kv.key = m_key.as_u64;
2829
2830   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2831     {
2832       /* Try address only mapping */
2833       m_key.port = 0;
2834       m_key.protocol = 0;
2835       kv.key = m_key.as_u64;
2836       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2837         return 1;
2838     }
2839
2840   m = pool_elt_at_index (sm->static_mappings, value.value);
2841
2842   if (by_external)
2843     {
2844       if (is_lb_static_mapping (m))
2845         {
2846           if (PREDICT_FALSE (lb != 0))
2847             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2848           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2849                                                           match.addr,
2850                                                           match.protocol,
2851                                                           match.port,
2852                                                           &backend_index))
2853             {
2854               local = pool_elt_at_index (m->locals, backend_index);
2855               mapping->addr = local->addr;
2856               mapping->port = clib_host_to_net_u16 (local->port);
2857               mapping->fib_index = local->fib_index;
2858               goto end;
2859             }
2860           // pick locals matching this worker
2861           if (PREDICT_FALSE (sm->num_workers > 1))
2862             {
2863               u32 thread_index = vlib_get_thread_index ();
2864               /* *INDENT-OFF* */
2865               pool_foreach_index (i, m->locals,
2866               ({
2867                 local = pool_elt_at_index (m->locals, i);
2868
2869                 ip4_header_t ip = {
2870                   .src_address = local->addr,
2871                 };
2872
2873                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2874                     thread_index)
2875                   {
2876                     vec_add1 (tmp, i);
2877                   }
2878               }));
2879               /* *INDENT-ON* */
2880               ASSERT (vec_len (tmp) != 0);
2881             }
2882           else
2883             {
2884               /* *INDENT-OFF* */
2885               pool_foreach_index (i, m->locals,
2886               ({
2887                 vec_add1 (tmp, i);
2888               }));
2889               /* *INDENT-ON* */
2890             }
2891           hi = vec_len (tmp) - 1;
2892           local = pool_elt_at_index (m->locals, tmp[hi]);
2893           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2894           while (lo < hi)
2895             {
2896               mid = ((hi - lo) >> 1) + lo;
2897               local = pool_elt_at_index (m->locals, tmp[mid]);
2898               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2899             }
2900           local = pool_elt_at_index (m->locals, tmp[lo]);
2901           if (!(local->prefix >= rand))
2902             return 1;
2903           mapping->addr = local->addr;
2904           mapping->port = clib_host_to_net_u16 (local->port);
2905           mapping->fib_index = local->fib_index;
2906           if (m->affinity)
2907             {
2908               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2909                                                 match.protocol, match.port,
2910                                                 tmp[lo], m->affinity,
2911                                                 m->affinity_per_service_list_head_index))
2912                 nat_elog_info ("create affinity record failed");
2913             }
2914           vec_free (tmp);
2915         }
2916       else
2917         {
2918           if (PREDICT_FALSE (lb != 0))
2919             *lb = NO_LB_NAT;
2920           mapping->fib_index = m->fib_index;
2921           mapping->addr = m->local_addr;
2922           /* Address only mapping doesn't change port */
2923           mapping->port = is_addr_only_static_mapping (m) ? match.port
2924             : clib_host_to_net_u16 (m->local_port);
2925         }
2926       mapping->protocol = m->proto;
2927     }
2928   else
2929     {
2930       mapping->addr = m->external_addr;
2931       /* Address only mapping doesn't change port */
2932       mapping->port = is_addr_only_static_mapping (m) ? match.port
2933         : clib_host_to_net_u16 (m->external_port);
2934       mapping->fib_index = sm->outside_fib_index;
2935     }
2936
2937 end:
2938   if (PREDICT_FALSE (is_addr_only != 0))
2939     *is_addr_only = is_addr_only_static_mapping (m);
2940
2941   if (PREDICT_FALSE (twice_nat != 0))
2942     *twice_nat = m->twice_nat;
2943
2944   if (PREDICT_FALSE (is_identity_nat != 0))
2945     *is_identity_nat = is_identity_static_mapping (m);
2946
2947   return 0;
2948 }
2949
2950 static_always_inline u16
2951 snat_random_port (u16 min, u16 max)
2952 {
2953   snat_main_t *sm = &snat_main;
2954   return min + random_u32 (&sm->random_seed) /
2955     (random_u32_max () / (max - min + 1) + 1);
2956 }
2957
2958 int
2959 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2960                                      u32 fib_index,
2961                                      u32 thread_index,
2962                                      snat_session_key_t * k,
2963                                      u16 port_per_thread,
2964                                      u32 snat_thread_index)
2965 {
2966   snat_main_t *sm = &snat_main;
2967
2968   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2969                                   port_per_thread, snat_thread_index);
2970 }
2971
2972 static int
2973 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2974                                  u32 fib_index,
2975                                  u32 thread_index,
2976                                  snat_session_key_t * k,
2977                                  u16 port_per_thread, u32 snat_thread_index)
2978 {
2979   int i;
2980   snat_address_t *a, *ga = 0;
2981   u32 portnum;
2982
2983   for (i = 0; i < vec_len (addresses); i++)
2984     {
2985       a = addresses + i;
2986       switch (k->protocol)
2987         {
2988 #define _(N, j, n, s) \
2989         case SNAT_PROTOCOL_##N: \
2990           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2991             { \
2992               if (a->fib_index == fib_index) \
2993                 { \
2994                   while (1) \
2995                     { \
2996                       portnum = (port_per_thread * \
2997                         snat_thread_index) + \
2998                         snat_random_port(1, port_per_thread) + 1024; \
2999                       if (a->busy_##n##_port_refcounts[portnum]) \
3000                         continue; \
3001                       --a->busy_##n##_port_refcounts[portnum]; \
3002                       a->busy_##n##_ports_per_thread[thread_index]++; \
3003                       a->busy_##n##_ports++; \
3004                       k->addr = a->addr; \
3005                       k->port = clib_host_to_net_u16(portnum); \
3006                       return 0; \
3007                     } \
3008                 } \
3009               else if (a->fib_index == ~0) \
3010                 { \
3011                   ga = a; \
3012                 } \
3013             } \
3014           break;
3015           foreach_snat_protocol
3016 #undef _
3017         default:
3018           nat_elog_info ("unknown protocol");
3019           return 1;
3020         }
3021
3022     }
3023
3024   if (ga)
3025     {
3026       a = ga;
3027       switch (k->protocol)
3028         {
3029 #define _(N, j, n, s) \
3030         case SNAT_PROTOCOL_##N: \
3031           while (1) \
3032             { \
3033               portnum = (port_per_thread * \
3034                 snat_thread_index) + \
3035                 snat_random_port(1, port_per_thread) + 1024; \
3036               if (a->busy_##n##_port_refcounts[portnum]) \
3037                 continue; \
3038               ++a->busy_##n##_port_refcounts[portnum]; \
3039               a->busy_##n##_ports_per_thread[thread_index]++; \
3040               a->busy_##n##_ports++; \
3041               k->addr = a->addr; \
3042               k->port = clib_host_to_net_u16(portnum); \
3043               return 0; \
3044             }
3045           break;
3046           foreach_snat_protocol
3047 #undef _
3048         default:
3049           nat_elog_info ("unknown protocol");
3050           return 1;
3051         }
3052     }
3053
3054   /* Totally out of translations to use... */
3055   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3056   return 1;
3057 }
3058
3059 static int
3060 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
3061                               u32 fib_index,
3062                               u32 thread_index,
3063                               snat_session_key_t * k,
3064                               u16 port_per_thread, u32 snat_thread_index)
3065 {
3066   snat_main_t *sm = &snat_main;
3067   snat_address_t *a = addresses;
3068   u16 m, ports, portnum, A, j;
3069   m = 16 - (sm->psid_offset + sm->psid_length);
3070   ports = (1 << (16 - sm->psid_length)) - (1 << m);
3071
3072   if (!vec_len (addresses))
3073     goto exhausted;
3074
3075   switch (k->protocol)
3076     {
3077 #define _(N, i, n, s) \
3078     case SNAT_PROTOCOL_##N: \
3079       if (a->busy_##n##_ports < ports) \
3080         { \
3081           while (1) \
3082             { \
3083               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3084               j = snat_random_port(0, pow2_mask(m)); \
3085               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3086               if (a->busy_##n##_port_refcounts[portnum]) \
3087                 continue; \
3088               ++a->busy_##n##_port_refcounts[portnum]; \
3089               a->busy_##n##_ports++; \
3090               k->addr = a->addr; \
3091               k->port = clib_host_to_net_u16 (portnum); \
3092               return 0; \
3093             } \
3094         } \
3095       break;
3096       foreach_snat_protocol
3097 #undef _
3098     default:
3099       nat_elog_info ("unknown protocol");
3100       return 1;
3101     }
3102
3103 exhausted:
3104   /* Totally out of translations to use... */
3105   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3106   return 1;
3107 }
3108
3109 static int
3110 nat_alloc_addr_and_port_range (snat_address_t * addresses,
3111                                u32 fib_index,
3112                                u32 thread_index,
3113                                snat_session_key_t * k,
3114                                u16 port_per_thread, u32 snat_thread_index)
3115 {
3116   snat_main_t *sm = &snat_main;
3117   snat_address_t *a = addresses;
3118   u16 portnum, ports;
3119
3120   ports = sm->end_port - sm->start_port + 1;
3121
3122   if (!vec_len (addresses))
3123     goto exhausted;
3124
3125   switch (k->protocol)
3126     {
3127 #define _(N, i, n, s) \
3128     case SNAT_PROTOCOL_##N: \
3129       if (a->busy_##n##_ports < ports) \
3130         { \
3131           while (1) \
3132             { \
3133               portnum = snat_random_port(sm->start_port, sm->end_port); \
3134               if (a->busy_##n##_port_refcounts[portnum]) \
3135                 continue; \
3136               ++a->busy_##n##_port_refcounts[portnum]; \
3137               a->busy_##n##_ports++; \
3138               k->addr = a->addr; \
3139               k->port = clib_host_to_net_u16 (portnum); \
3140               return 0; \
3141             } \
3142         } \
3143       break;
3144       foreach_snat_protocol
3145 #undef _
3146     default:
3147       nat_elog_info ("unknown protocol");
3148       return 1;
3149     }
3150
3151 exhausted:
3152   /* Totally out of translations to use... */
3153   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3154   return 1;
3155 }
3156
3157 void
3158 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3159 {
3160   dpo_id_t dpo_v4 = DPO_INVALID;
3161   fib_prefix_t pfx = {
3162     .fp_proto = FIB_PROTOCOL_IP4,
3163     .fp_len = 32,
3164     .fp_addr.ip4.as_u32 = addr.as_u32,
3165   };
3166
3167   if (is_add)
3168     {
3169       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3170       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3171                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3172       dpo_reset (&dpo_v4);
3173     }
3174   else
3175     {
3176       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3177     }
3178 }
3179
3180 u8 *
3181 format_session_kvp (u8 * s, va_list * args)
3182 {
3183   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3184   snat_session_key_t k;
3185
3186   k.as_u64 = v->key;
3187
3188   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
3189
3190   return s;
3191 }
3192
3193 u8 *
3194 format_static_mapping_kvp (u8 * s, va_list * args)
3195 {
3196   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3197   snat_session_key_t k;
3198
3199   k.as_u64 = v->key;
3200
3201   s = format (s, "%U static-mapping-index %llu",
3202               format_static_mapping_key, &k, v->value);
3203
3204   return s;
3205 }
3206
3207 u8 *
3208 format_user_kvp (u8 * s, va_list * args)
3209 {
3210   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3211   snat_user_key_t k;
3212
3213   k.as_u64 = v->key;
3214
3215   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3216               k.fib_index, v->value);
3217
3218   return s;
3219 }
3220
3221 u8 *
3222 format_ed_session_kvp (u8 * s, va_list * args)
3223 {
3224   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3225
3226   u8 proto;
3227   u16 r_port, l_port;
3228   ip4_address_t l_addr, r_addr;
3229   u32 fib_index;
3230
3231   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
3232   s =
3233     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3234             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
3235             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
3236             format_ip_protocol, proto, fib_index, v->value);
3237
3238   return s;
3239 }
3240
3241 static u32
3242 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3243                            u8 is_output)
3244 {
3245   snat_main_t *sm = &snat_main;
3246   u32 next_worker_index = 0;
3247   u32 hash;
3248
3249   next_worker_index = sm->first_worker_index;
3250   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3251     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3252
3253   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3254     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3255   else
3256     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3257
3258   return next_worker_index;
3259 }
3260
3261 static u32
3262 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3263                            u32 rx_fib_index0, u8 is_output)
3264 {
3265   snat_main_t *sm = &snat_main;
3266   udp_header_t *udp;
3267   u16 port;
3268   snat_session_key_t m_key;
3269   clib_bihash_kv_8_8_t kv, value;
3270   snat_static_mapping_t *m;
3271   u32 proto;
3272   u32 next_worker_index = 0;
3273
3274   /* first try static mappings without port */
3275   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3276     {
3277       m_key.addr = ip0->dst_address;
3278       m_key.port = 0;
3279       m_key.protocol = 0;
3280       m_key.fib_index = rx_fib_index0;
3281       kv.key = m_key.as_u64;
3282       if (!clib_bihash_search_8_8
3283           (&sm->static_mapping_by_external, &kv, &value))
3284         {
3285           m = pool_elt_at_index (sm->static_mappings, value.value);
3286           return m->workers[0];
3287         }
3288     }
3289
3290   proto = ip_proto_to_snat_proto (ip0->protocol);
3291   udp = ip4_next_header (ip0);
3292   port = udp->dst_port;
3293
3294   /* unknown protocol */
3295   if (PREDICT_FALSE (proto == ~0))
3296     {
3297       /* use current thread */
3298       return vlib_get_thread_index ();
3299     }
3300
3301   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3302     {
3303       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3304       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3305       if (!icmp_type_is_error_message
3306           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3307         port = vnet_buffer (b)->ip.reass.l4_src_port;
3308       else
3309         {
3310           /* if error message, then it's not fragmented and we can access it */
3311           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3312           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3313           void *l4_header = ip4_next_header (inner_ip);
3314           switch (proto)
3315             {
3316             case SNAT_PROTOCOL_ICMP:
3317               icmp = (icmp46_header_t *) l4_header;
3318               echo = (icmp_echo_header_t *) (icmp + 1);
3319               port = echo->identifier;
3320               break;
3321             case SNAT_PROTOCOL_UDP:
3322             case SNAT_PROTOCOL_TCP:
3323               port = ((tcp_udp_header_t *) l4_header)->src_port;
3324               break;
3325             default:
3326               return vlib_get_thread_index ();
3327             }
3328         }
3329     }
3330
3331   /* try static mappings with port */
3332   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3333     {
3334       m_key.addr = ip0->dst_address;
3335       m_key.port = clib_net_to_host_u16 (port);
3336       m_key.protocol = proto;
3337       m_key.fib_index = rx_fib_index0;
3338       kv.key = m_key.as_u64;
3339       if (!clib_bihash_search_8_8
3340           (&sm->static_mapping_by_external, &kv, &value))
3341         {
3342           m = pool_elt_at_index (sm->static_mappings, value.value);
3343           return m->workers[0];
3344         }
3345     }
3346
3347   /* worker by outside port */
3348   next_worker_index = sm->first_worker_index;
3349   next_worker_index +=
3350     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3351   return next_worker_index;
3352 }
3353
3354 static u32
3355 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3356                                u8 is_output)
3357 {
3358   snat_main_t *sm = &snat_main;
3359   u32 next_worker_index = sm->first_worker_index;
3360   u32 hash;
3361
3362   clib_bihash_kv_16_8_t kv16, value16;
3363   snat_main_per_thread_data_t *tsm;
3364   udp_header_t *udp;
3365
3366   if (PREDICT_FALSE (is_output))
3367     {
3368       u32 fib_index = sm->outside_fib_index;
3369       nat_outside_fib_t *outside_fib;
3370       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3371       fib_prefix_t pfx = {
3372         .fp_proto = FIB_PROTOCOL_IP4,
3373         .fp_len = 32,
3374         .fp_addr = {
3375                     .ip4.as_u32 = ip->dst_address.as_u32,
3376                     }
3377         ,
3378       };
3379
3380       udp = ip4_next_header (ip);
3381
3382       switch (vec_len (sm->outside_fibs))
3383         {
3384         case 0:
3385           fib_index = sm->outside_fib_index;
3386           break;
3387         case 1:
3388           fib_index = sm->outside_fibs[0].fib_index;
3389           break;
3390         default:
3391             /* *INDENT-OFF* */
3392             vec_foreach (outside_fib, sm->outside_fibs)
3393               {
3394                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3395                 if (FIB_NODE_INDEX_INVALID != fei)
3396                   {
3397                     if (fib_entry_get_resolving_interface (fei) != ~0)
3398                       {
3399                         fib_index = outside_fib->fib_index;
3400                         break;
3401                       }
3402                   }
3403               }
3404             /* *INDENT-ON* */
3405           break;
3406         }
3407
3408       make_ed_kv (&ip->src_address, &ip->dst_address,
3409                   ip->protocol, fib_index, udp->src_port, udp->dst_port,
3410                   ~0ULL, &kv16);
3411
3412       /* *INDENT-OFF* */
3413       vec_foreach (tsm, sm->per_thread_data)
3414         {
3415           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3416                                                       &kv16, &value16)))
3417             {
3418               next_worker_index += tsm->thread_index;
3419
3420               nat_elog_debug_handoff (
3421                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3422                 next_worker_index, fib_index,
3423                 clib_net_to_host_u32 (ip->src_address.as_u32),
3424                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3425
3426               return next_worker_index;
3427             }
3428         }
3429       /* *INDENT-ON* */
3430     }
3431
3432   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3433     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3434
3435   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3436     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3437   else
3438     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3439
3440   if (PREDICT_TRUE (!is_output))
3441     {
3442       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3443                               next_worker_index, rx_fib_index,
3444                               clib_net_to_host_u32 (ip->src_address.as_u32),
3445                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3446     }
3447   else
3448     {
3449       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3450                               next_worker_index, rx_fib_index,
3451                               clib_net_to_host_u32 (ip->src_address.as_u32),
3452                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3453     }
3454
3455   return next_worker_index;
3456 }
3457
3458 static u32
3459 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3460                                u32 rx_fib_index, u8 is_output)
3461 {
3462   snat_main_t *sm = &snat_main;
3463   clib_bihash_kv_8_8_t kv, value;
3464   clib_bihash_kv_16_8_t kv16, value16;
3465   snat_main_per_thread_data_t *tsm;
3466
3467   u32 proto, next_worker_index = 0;
3468   udp_header_t *udp;
3469   u16 port;
3470   snat_static_mapping_t *m;
3471   u32 hash;
3472
3473   proto = ip_proto_to_snat_proto (ip->protocol);
3474
3475   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3476     {
3477       udp = ip4_next_header (ip);
3478
3479       make_ed_kv (&ip->dst_address, &ip->src_address,
3480                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port,
3481                   ~0ULL, &kv16);
3482
3483       /* *INDENT-OFF* */
3484       vec_foreach (tsm, sm->per_thread_data)
3485         {
3486           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3487                                                       &kv16, &value16)))
3488             {
3489               next_worker_index = sm->first_worker_index + tsm->thread_index;
3490               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3491                           next_worker_index, rx_fib_index,
3492                           clib_net_to_host_u32 (ip->src_address.as_u32),
3493                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3494               return next_worker_index;
3495             }
3496           }
3497         /* *INDENT-ON* */
3498     }
3499   else if (proto == SNAT_PROTOCOL_ICMP)
3500     {
3501       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0ULL, 0, 0, 0, &kv16))
3502         {
3503           /* *INDENT-OFF* */
3504           vec_foreach (tsm, sm->per_thread_data)
3505             {
3506               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3507                                                           &kv16, &value16)))
3508                 {
3509                   next_worker_index = sm->first_worker_index +
3510                                       tsm->thread_index;
3511                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3512                               next_worker_index, rx_fib_index,
3513                               clib_net_to_host_u32 (ip->src_address.as_u32),
3514                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3515                   return next_worker_index;
3516                 }
3517             }
3518           /* *INDENT-ON* */
3519         }
3520     }
3521
3522   /* first try static mappings without port */
3523   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3524     {
3525       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3526       if (!clib_bihash_search_8_8
3527           (&sm->static_mapping_by_external, &kv, &value))
3528         {
3529           m = pool_elt_at_index (sm->static_mappings, value.value);
3530           next_worker_index = m->workers[0];
3531           goto done;
3532         }
3533     }
3534
3535   /* unknown protocol */
3536   if (PREDICT_FALSE (proto == ~0))
3537     {
3538       /* use current thread */
3539       next_worker_index = vlib_get_thread_index ();
3540       goto done;
3541     }
3542
3543   udp = ip4_next_header (ip);
3544   port = udp->dst_port;
3545
3546   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3547     {
3548       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3549       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3550       if (!icmp_type_is_error_message
3551           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3552         port = vnet_buffer (b)->ip.reass.l4_src_port;
3553       else
3554         {
3555           /* if error message, then it's not fragmented and we can access it */
3556           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3557           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3558           void *l4_header = ip4_next_header (inner_ip);
3559           switch (proto)
3560             {
3561             case SNAT_PROTOCOL_ICMP:
3562               icmp = (icmp46_header_t *) l4_header;
3563               echo = (icmp_echo_header_t *) (icmp + 1);
3564               port = echo->identifier;
3565               break;
3566             case SNAT_PROTOCOL_UDP:
3567             case SNAT_PROTOCOL_TCP:
3568               port = ((tcp_udp_header_t *) l4_header)->src_port;
3569               break;
3570             default:
3571               next_worker_index = vlib_get_thread_index ();
3572               goto done;
3573             }
3574         }
3575     }
3576
3577   /* try static mappings with port */
3578   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3579     {
3580       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3581                   clib_net_to_host_u16 (port));
3582       if (!clib_bihash_search_8_8
3583           (&sm->static_mapping_by_external, &kv, &value))
3584         {
3585           m = pool_elt_at_index (sm->static_mappings, value.value);
3586           if (!is_lb_static_mapping (m))
3587             {
3588               next_worker_index = m->workers[0];
3589               goto done;
3590             }
3591
3592           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3593             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3594
3595           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3596             next_worker_index =
3597               m->workers[hash & (_vec_len (m->workers) - 1)];
3598           else
3599             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3600           goto done;
3601         }
3602     }
3603
3604   /* worker by outside port */
3605   next_worker_index = sm->first_worker_index;
3606   next_worker_index +=
3607     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3608
3609 done:
3610   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3611                           clib_net_to_host_u32 (ip->src_address.as_u32),
3612                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3613   return next_worker_index;
3614 }
3615
3616 void
3617 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3618                 ip4_address_t * out_addr, u16 out_port,
3619                 ip4_address_t * eh_addr, u16 eh_port,
3620                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3621                 u32 fib_index, u16 flags, u32 thread_index)
3622 {
3623   snat_main_t *sm = &snat_main;
3624   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3625   snat_session_key_t key;
3626   snat_user_t *u;
3627   snat_session_t *s;
3628   clib_bihash_kv_8_8_t kv;
3629   vlib_main_t *vm = vlib_get_main ();
3630   f64 now = vlib_time_now (vm);
3631   nat_outside_fib_t *outside_fib;
3632   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3633   fib_prefix_t pfx = {
3634     .fp_proto = FIB_PROTOCOL_IP4,
3635     .fp_len = 32,
3636     .fp_addr = {
3637                 .ip4.as_u32 = eh_addr->as_u32,
3638                 },
3639   };
3640
3641   key.addr.as_u32 = out_addr->as_u32;
3642   key.port = out_port;
3643   key.protocol = proto;
3644
3645   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3646     {
3647       if (nat_set_outside_address_and_port
3648           (sm->addresses, thread_index, &key))
3649         return;
3650     }
3651
3652   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3653   if (!u)
3654     return;
3655
3656   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3657   if (!s)
3658     return;
3659
3660   s->last_heard = now;
3661   s->flags = flags;
3662   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3663   s->ext_host_port = eh_port;
3664   user_session_increment (sm, u, snat_is_session_static (s));
3665   switch (vec_len (sm->outside_fibs))
3666     {
3667     case 0:
3668       key.fib_index = sm->outside_fib_index;
3669       break;
3670     case 1:
3671       key.fib_index = sm->outside_fibs[0].fib_index;
3672       break;
3673     default:
3674       /* *INDENT-OFF* */
3675       vec_foreach (outside_fib, sm->outside_fibs)
3676         {
3677           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3678           if (FIB_NODE_INDEX_INVALID != fei)
3679             {
3680               if (fib_entry_get_resolving_interface (fei) != ~0)
3681                 {
3682                   key.fib_index = outside_fib->fib_index;
3683                   break;
3684                 }
3685             }
3686         }
3687       /* *INDENT-ON* */
3688       break;
3689     }
3690   s->out2in = key;
3691   kv.key = key.as_u64;
3692   kv.value = s - tsm->sessions;
3693   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3694     nat_elog_warn ("out2in key add failed");
3695
3696   key.addr.as_u32 = in_addr->as_u32;
3697   key.port = in_port;
3698   key.fib_index = fib_index;
3699   s->in2out = key;
3700   kv.key = key.as_u64;
3701   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3702     nat_elog_warn ("in2out key add failed");
3703 }
3704
3705 void
3706 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3707                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3708                 u32 ti)
3709 {
3710   snat_main_t *sm = &snat_main;
3711   snat_session_key_t key;
3712   clib_bihash_kv_8_8_t kv, value;
3713   u32 thread_index;
3714   snat_session_t *s;
3715   snat_main_per_thread_data_t *tsm;
3716
3717   if (sm->num_workers > 1)
3718     thread_index =
3719       sm->first_worker_index +
3720       (sm->workers[(clib_net_to_host_u16 (out_port) -
3721                     1024) / sm->port_per_thread]);
3722   else
3723     thread_index = sm->num_workers;
3724   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3725
3726   key.addr.as_u32 = out_addr->as_u32;
3727   key.port = out_port;
3728   key.protocol = proto;
3729   key.fib_index = fib_index;
3730   kv.key = key.as_u64;
3731   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3732     return;
3733
3734   s = pool_elt_at_index (tsm->sessions, value.value);
3735   nat_free_session_data (sm, s, thread_index, 1);
3736   nat44_delete_session (sm, s, thread_index);
3737 }
3738
3739 void
3740 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3741                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3742                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3743 {
3744   snat_main_t *sm = &snat_main;
3745   snat_session_key_t key;
3746   clib_bihash_kv_8_8_t kv, value;
3747   snat_session_t *s;
3748   snat_main_per_thread_data_t *tsm;
3749
3750   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3751
3752   key.addr.as_u32 = out_addr->as_u32;
3753   key.port = out_port;
3754   key.protocol = proto;
3755   key.fib_index = fib_index;
3756   kv.key = key.as_u64;
3757   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3758     return;
3759
3760   s = pool_elt_at_index (tsm->sessions, value.value);
3761   s->total_pkts = total_pkts;
3762   s->total_bytes = total_bytes;
3763 }
3764
3765 void
3766 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3767                    ip4_address_t * out_addr, u16 out_port,
3768                    ip4_address_t * eh_addr, u16 eh_port,
3769                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3770                    u32 fib_index, u16 flags, u32 thread_index)
3771 {
3772   snat_main_t *sm = &snat_main;
3773   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3774   snat_session_key_t key;
3775   snat_session_t *s;
3776   clib_bihash_kv_16_8_t kv;
3777   vlib_main_t *vm = vlib_get_main ();
3778   f64 now = vlib_time_now (vm);
3779   nat_outside_fib_t *outside_fib;
3780   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3781   fib_prefix_t pfx = {
3782     .fp_proto = FIB_PROTOCOL_IP4,
3783     .fp_len = 32,
3784     .fp_addr = {
3785                 .ip4.as_u32 = eh_addr->as_u32,
3786                 },
3787   };
3788
3789   key.addr.as_u32 = out_addr->as_u32;
3790   key.port = out_port;
3791   key.protocol = proto;
3792
3793   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3794     {
3795       if (nat_set_outside_address_and_port
3796           (sm->addresses, thread_index, &key))
3797         return;
3798     }
3799
3800   key.addr.as_u32 = ehn_addr->as_u32;
3801   key.port = ehn_port;
3802   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3803     {
3804       if (nat_set_outside_address_and_port
3805           (sm->twice_nat_addresses, thread_index, &key))
3806         return;
3807     }
3808
3809   s = nat_ed_session_alloc (sm, thread_index, now);
3810   if (!s)
3811     return;
3812
3813   s->last_heard = now;
3814   s->flags = flags;
3815   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3816   s->ext_host_nat_port = s->ext_host_port = eh_port;
3817   if (is_twice_nat_session (s))
3818     {
3819       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3820       s->ext_host_nat_port = ehn_port;
3821     }
3822   switch (vec_len (sm->outside_fibs))
3823     {
3824     case 0:
3825       key.fib_index = sm->outside_fib_index;
3826       break;
3827     case 1:
3828       key.fib_index = sm->outside_fibs[0].fib_index;
3829       break;
3830     default:
3831       /* *INDENT-OFF* */
3832       vec_foreach (outside_fib, sm->outside_fibs)
3833         {
3834           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3835           if (FIB_NODE_INDEX_INVALID != fei)
3836             {
3837               if (fib_entry_get_resolving_interface (fei) != ~0)
3838                 {
3839                   key.fib_index = outside_fib->fib_index;
3840                   break;
3841                 }
3842             }
3843         }
3844       /* *INDENT-ON* */
3845       break;
3846     }
3847   key.addr.as_u32 = out_addr->as_u32;
3848   key.port = out_port;
3849   s->out2in = key;
3850   kv.value = s - tsm->sessions;
3851
3852   key.addr.as_u32 = in_addr->as_u32;
3853   key.port = in_port;
3854   key.fib_index = fib_index;
3855   s->in2out = key;
3856
3857   make_ed_kv (in_addr, &s->ext_host_nat_addr,
3858               snat_proto_to_ip_proto (proto), fib_index, in_port,
3859               s->ext_host_nat_port, s - tsm->sessions, &kv);
3860   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3861     nat_elog_warn ("in2out key add failed");
3862
3863   make_ed_kv (out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3864               s->out2in.fib_index, out_port, eh_port, s - tsm->sessions, &kv);
3865   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3866     nat_elog_warn ("out2in key add failed");
3867 }
3868
3869 void
3870 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3871                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3872                    u32 fib_index, u32 ti)
3873 {
3874   snat_main_t *sm = &snat_main;
3875   clib_bihash_kv_16_8_t kv, value;
3876   u32 thread_index;
3877   snat_session_t *s;
3878   snat_main_per_thread_data_t *tsm;
3879
3880   if (sm->num_workers > 1)
3881     thread_index =
3882       sm->first_worker_index +
3883       (sm->workers[(clib_net_to_host_u16 (out_port) -
3884                     1024) / sm->port_per_thread]);
3885   else
3886     thread_index = sm->num_workers;
3887   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3888
3889   make_ed_kv (out_addr, eh_addr, proto, fib_index, out_port, eh_port, ~0ULL,
3890               &kv);
3891   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3892     return;
3893
3894   s = pool_elt_at_index (tsm->sessions, value.value);
3895   nat_free_session_data (sm, s, thread_index, 1);
3896   nat44_delete_session (sm, s, thread_index);
3897 }
3898
3899 void
3900 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3901                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3902                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3903                    u32 thread_index)
3904 {
3905   snat_main_t *sm = &snat_main;
3906   clib_bihash_kv_16_8_t kv, value;
3907   snat_session_t *s;
3908   snat_main_per_thread_data_t *tsm;
3909
3910   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3911
3912   make_ed_kv (out_addr, eh_addr, proto, fib_index, out_port, eh_port, ~0ULL,
3913               &kv);
3914   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3915     return;
3916
3917   s = pool_elt_at_index (tsm->sessions, value.value);
3918   s->total_pkts = total_pkts;
3919   s->total_bytes = total_bytes;
3920 }
3921
3922 void
3923 nat44_db_init (snat_main_per_thread_data_t * tsm)
3924 {
3925   snat_main_t *sm = &snat_main;
3926
3927   pool_alloc (tsm->sessions, sm->max_translations);
3928   pool_alloc (tsm->global_lru_pool, sm->max_translations);
3929
3930   dlist_elt_t *head;
3931   pool_get (tsm->global_lru_pool, head);
3932   tsm->global_lru_head_index = head - tsm->global_lru_pool;
3933   clib_dlist_init (tsm->global_lru_pool, tsm->global_lru_head_index);
3934
3935   if (sm->endpoint_dependent)
3936     {
3937       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3938                              sm->translation_buckets,
3939                              sm->translation_memory_size);
3940       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3941                                           format_ed_session_kvp);
3942       clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3943                              sm->translation_buckets,
3944                              sm->translation_memory_size);
3945       clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3946                                           format_ed_session_kvp);
3947     }
3948   else
3949     {
3950       clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3951                             sm->translation_buckets,
3952                             sm->translation_memory_size);
3953       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
3954       clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3955                             sm->translation_buckets,
3956                             sm->translation_memory_size);
3957       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
3958     }
3959
3960   // TODO: resolve static mappings (put only to !ED)
3961   pool_alloc (tsm->list_pool, sm->max_translations);
3962   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets,
3963                         sm->user_memory_size);
3964   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3965 }
3966
3967 void
3968 nat44_db_free (snat_main_per_thread_data_t * tsm)
3969 {
3970   snat_main_t *sm = &snat_main;
3971
3972   pool_free (tsm->sessions);
3973   pool_free (tsm->global_lru_pool);
3974
3975   if (sm->endpoint_dependent)
3976     {
3977       clib_bihash_free_16_8 (&tsm->in2out_ed);
3978       clib_bihash_free_16_8 (&tsm->out2in_ed);
3979     }
3980   else
3981     {
3982       clib_bihash_free_8_8 (&tsm->in2out);
3983       clib_bihash_free_8_8 (&tsm->out2in);
3984     }
3985
3986   // TODO: resolve static mappings (put only to !ED)
3987   pool_free (tsm->users);
3988   pool_free (tsm->list_pool);
3989   clib_bihash_free_8_8 (&tsm->user_hash);
3990 }
3991
3992 static clib_error_t *
3993 snat_config (vlib_main_t * vm, unformat_input_t * input)
3994 {
3995   snat_main_t *sm = &snat_main;
3996   nat66_main_t *nm = &nat66_main;
3997   snat_main_per_thread_data_t *tsm;
3998
3999   u32 static_mapping_buckets = 1024;
4000   uword static_mapping_memory_size = 64 << 20;
4001
4002   u32 nat64_bib_buckets = 1024;
4003   u32 nat64_bib_memory_size = 128 << 20;
4004
4005   u32 nat64_st_buckets = 2048;
4006   uword nat64_st_memory_size = 256 << 20;
4007
4008   u32 user_buckets = 128;
4009   uword user_memory_size = 64 << 20;
4010   u32 translation_buckets = 1024;
4011   uword translation_memory_size = 128 << 20;
4012
4013   u32 max_translations_per_user = ~0;
4014
4015   u32 outside_vrf_id = 0;
4016   u32 outside_ip6_vrf_id = 0;
4017   u32 inside_vrf_id = 0;
4018   u8 static_mapping_only = 0;
4019   u8 static_mapping_connection_tracking = 0;
4020
4021   // configurable timeouts
4022   u32 udp_timeout = SNAT_UDP_TIMEOUT;
4023   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
4024   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
4025   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
4026
4027   sm->deterministic = 0;
4028   sm->out2in_dpo = 0;
4029   sm->endpoint_dependent = 0;
4030
4031   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
4032     {
4033       if (unformat
4034           (input, "translation hash buckets %d", &translation_buckets))
4035         ;
4036       else if (unformat (input, "udp timeout %d", &udp_timeout))
4037         ;
4038       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
4039         ;
4040       else if (unformat (input, "tcp transitory timeout %d",
4041                          &tcp_transitory_timeout));
4042       else if (unformat (input, "tcp established timeout %d",
4043                          &tcp_established_timeout));
4044       else if (unformat (input, "translation hash memory %d",
4045                          &translation_memory_size));
4046       else if (unformat (input, "user hash buckets %d", &user_buckets))
4047         ;
4048       else if (unformat (input, "user hash memory %d", &user_memory_size))
4049         ;
4050       else if (unformat (input, "max translations per user %d",
4051                          &max_translations_per_user))
4052         ;
4053       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
4054         ;
4055       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
4056         ;
4057       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
4058         ;
4059       else if (unformat (input, "static mapping only"))
4060         {
4061           static_mapping_only = 1;
4062           if (unformat (input, "connection tracking"))
4063             static_mapping_connection_tracking = 1;
4064         }
4065       else if (unformat (input, "deterministic"))
4066         sm->deterministic = 1;
4067       else if (unformat (input, "nat64 bib hash buckets %d",
4068                          &nat64_bib_buckets))
4069         ;
4070       else if (unformat (input, "nat64 bib hash memory %d",
4071                          &nat64_bib_memory_size))
4072         ;
4073       else
4074         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
4075         ;
4076       else if (unformat (input, "nat64 st hash memory %d",
4077                          &nat64_st_memory_size))
4078         ;
4079       else if (unformat (input, "out2in dpo"))
4080         sm->out2in_dpo = 1;
4081       //else if (unformat (input, "dslite ce"))
4082       //dslite_set_ce (dm, 1);
4083       else if (unformat (input, "endpoint-dependent"))
4084         sm->endpoint_dependent = 1;
4085       else
4086         return clib_error_return (0, "unknown input '%U'",
4087                                   format_unformat_error, input);
4088     }
4089
4090   if (sm->deterministic && sm->endpoint_dependent)
4091     return clib_error_return (0,
4092                               "deterministic and endpoint-dependent modes are mutually exclusive");
4093
4094   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
4095     return clib_error_return (0,
4096                               "static mapping only mode available only for simple nat");
4097
4098   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
4099     return clib_error_return (0,
4100                               "out2in dpo mode available only for simple nat");
4101
4102   /* optionally configurable timeouts for testing purposes */
4103   sm->udp_timeout = udp_timeout;
4104   sm->tcp_transitory_timeout = tcp_transitory_timeout;
4105   sm->tcp_established_timeout = tcp_established_timeout;
4106   sm->icmp_timeout = icmp_timeout;
4107
4108   sm->user_buckets = user_buckets;
4109   sm->user_memory_size = user_memory_size;
4110
4111   sm->translation_buckets = translation_buckets;
4112   sm->translation_memory_size = translation_memory_size;
4113   /* do not exceed load factor 10 */
4114   sm->max_translations = 10 * translation_buckets;
4115   vec_add1 (sm->max_translations_per_fib, sm->max_translations);
4116
4117   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4118     sm->max_translations : max_translations_per_user;
4119
4120   sm->outside_vrf_id = outside_vrf_id;
4121   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4122                                                              outside_vrf_id,
4123                                                              nat_fib_src_hi);
4124   nm->outside_vrf_id = outside_ip6_vrf_id;
4125   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
4126                                                              outside_ip6_vrf_id,
4127                                                              nat_fib_src_hi);
4128   sm->inside_vrf_id = inside_vrf_id;
4129   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4130                                                             inside_vrf_id,
4131                                                             nat_fib_src_hi);
4132   sm->static_mapping_only = static_mapping_only;
4133   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4134
4135   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4136                   nat64_st_memory_size);
4137
4138   if (sm->deterministic)
4139     {
4140       sm->in2out_node_index = snat_det_in2out_node.index;
4141       sm->in2out_output_node_index = ~0;
4142       sm->out2in_node_index = snat_det_out2in_node.index;
4143       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4144       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4145     }
4146   else
4147     {
4148       if (sm->endpoint_dependent)
4149         {
4150           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4151           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4152
4153           sm->in2out_node_index = nat44_ed_in2out_node.index;
4154           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4155           sm->out2in_node_index = nat44_ed_out2in_node.index;
4156
4157           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4158           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4159           nat_affinity_init (vm);
4160           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4161                        nat_ha_sref_ed_cb);
4162         }
4163       else
4164         {
4165           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4166           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4167
4168           sm->in2out_node_index = snat_in2out_node.index;
4169           sm->in2out_output_node_index = snat_in2out_output_node.index;
4170           sm->out2in_node_index = snat_out2in_node.index;
4171
4172           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4173           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4174           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4175         }
4176       if (!static_mapping_only ||
4177           (static_mapping_only && static_mapping_connection_tracking))
4178         {
4179           /* *INDENT-OFF* */
4180           vec_foreach (tsm, sm->per_thread_data)
4181             {
4182               nat44_db_init (tsm);
4183             }
4184           /* *INDENT-ON* */
4185         }
4186       else
4187         {
4188           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4189           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4190         }
4191       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4192                             "static_mapping_by_local", static_mapping_buckets,
4193                             static_mapping_memory_size);
4194       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4195                                          format_static_mapping_kvp);
4196
4197       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4198                             "static_mapping_by_external",
4199                             static_mapping_buckets,
4200                             static_mapping_memory_size);
4201       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4202                                          format_static_mapping_kvp);
4203     }
4204
4205   return 0;
4206 }
4207
4208 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4209
4210 static void
4211 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4212                                  uword opaque,
4213                                  u32 sw_if_index,
4214                                  ip4_address_t * address,
4215                                  u32 address_length,
4216                                  u32 if_address_index, u32 is_delete)
4217 {
4218   snat_main_t *sm = &snat_main;
4219   snat_static_map_resolve_t *rp;
4220   snat_static_mapping_t *m;
4221   snat_session_key_t m_key;
4222   clib_bihash_kv_8_8_t kv, value;
4223   int i, rv;
4224   ip4_address_t l_addr;
4225
4226   for (i = 0; i < vec_len (sm->to_resolve); i++)
4227     {
4228       rp = sm->to_resolve + i;
4229       if (rp->addr_only == 0)
4230         continue;
4231       if (rp->sw_if_index == sw_if_index)
4232         goto match;
4233     }
4234
4235   return;
4236
4237 match:
4238   m_key.addr.as_u32 = address->as_u32;
4239   m_key.port = rp->addr_only ? 0 : rp->e_port;
4240   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4241   m_key.fib_index = sm->outside_fib_index;
4242   kv.key = m_key.as_u64;
4243   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4244     m = 0;
4245   else
4246     m = pool_elt_at_index (sm->static_mappings, value.value);
4247
4248   if (!is_delete)
4249     {
4250       /* Don't trip over lease renewal, static config */
4251       if (m)
4252         return;
4253     }
4254   else
4255     {
4256       if (!m)
4257         return;
4258     }
4259
4260   /* Indetity mapping? */
4261   if (rp->l_addr.as_u32 == 0)
4262     l_addr.as_u32 = address[0].as_u32;
4263   else
4264     l_addr.as_u32 = rp->l_addr.as_u32;
4265   /* Add the static mapping */
4266   rv = snat_add_static_mapping (l_addr,
4267                                 address[0],
4268                                 rp->l_port,
4269                                 rp->e_port,
4270                                 rp->vrf_id,
4271                                 rp->addr_only, ~0 /* sw_if_index */ ,
4272                                 rp->proto, !is_delete, rp->twice_nat,
4273                                 rp->out2in_only, rp->tag, rp->identity_nat);
4274   if (rv)
4275     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4276 }
4277
4278 static void
4279 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4280                                        uword opaque,
4281                                        u32 sw_if_index,
4282                                        ip4_address_t * address,
4283                                        u32 address_length,
4284                                        u32 if_address_index, u32 is_delete)
4285 {
4286   snat_main_t *sm = &snat_main;
4287   snat_static_map_resolve_t *rp;
4288   ip4_address_t l_addr;
4289   int i, j;
4290   int rv;
4291   u8 twice_nat = 0;
4292   snat_address_t *addresses = sm->addresses;
4293
4294   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4295     {
4296       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4297         goto match;
4298     }
4299
4300   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4301     {
4302       twice_nat = 1;
4303       addresses = sm->twice_nat_addresses;
4304       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4305         goto match;
4306     }
4307
4308   return;
4309
4310 match:
4311   if (!is_delete)
4312     {
4313       /* Don't trip over lease renewal, static config */
4314       for (j = 0; j < vec_len (addresses); j++)
4315         if (addresses[j].addr.as_u32 == address->as_u32)
4316           return;
4317
4318       (void) snat_add_address (sm, address, ~0, twice_nat);
4319       /* Scan static map resolution vector */
4320       for (j = 0; j < vec_len (sm->to_resolve); j++)
4321         {
4322           rp = sm->to_resolve + j;
4323           if (rp->addr_only)
4324             continue;
4325           /* On this interface? */
4326           if (rp->sw_if_index == sw_if_index)
4327             {
4328               /* Indetity mapping? */
4329               if (rp->l_addr.as_u32 == 0)
4330                 l_addr.as_u32 = address[0].as_u32;
4331               else
4332                 l_addr.as_u32 = rp->l_addr.as_u32;
4333               /* Add the static mapping */
4334               rv = snat_add_static_mapping (l_addr,
4335                                             address[0],
4336                                             rp->l_port,
4337                                             rp->e_port,
4338                                             rp->vrf_id,
4339                                             rp->addr_only,
4340                                             ~0 /* sw_if_index */ ,
4341                                             rp->proto,
4342                                             rp->is_add, rp->twice_nat,
4343                                             rp->out2in_only, rp->tag,
4344                                             rp->identity_nat);
4345               if (rv)
4346                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4347                                     "i4", rv);
4348             }
4349         }
4350       return;
4351     }
4352   else
4353     {
4354       (void) snat_del_address (sm, address[0], 1, twice_nat);
4355       return;
4356     }
4357 }
4358
4359
4360 int
4361 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4362                             u8 twice_nat)
4363 {
4364   ip4_main_t *ip4_main = sm->ip4_main;
4365   ip4_address_t *first_int_addr;
4366   snat_static_map_resolve_t *rp;
4367   u32 *indices_to_delete = 0;
4368   int i, j;
4369   u32 *auto_add_sw_if_indices =
4370     twice_nat ? sm->
4371     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4372
4373   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4374     );
4375
4376   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4377     {
4378       if (auto_add_sw_if_indices[i] == sw_if_index)
4379         {
4380           if (is_del)
4381             {
4382               /* if have address remove it */
4383               if (first_int_addr)
4384                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4385               else
4386                 {
4387                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4388                     {
4389                       rp = sm->to_resolve + j;
4390                       if (rp->sw_if_index == sw_if_index)
4391                         vec_add1 (indices_to_delete, j);
4392                     }
4393                   if (vec_len (indices_to_delete))
4394                     {
4395                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4396                         vec_del1 (sm->to_resolve, j);
4397                       vec_free (indices_to_delete);
4398                     }
4399                 }
4400               if (twice_nat)
4401                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4402               else
4403                 vec_del1 (sm->auto_add_sw_if_indices, i);
4404             }
4405           else
4406             return VNET_API_ERROR_VALUE_EXIST;
4407
4408           return 0;
4409         }
4410     }
4411
4412   if (is_del)
4413     return VNET_API_ERROR_NO_SUCH_ENTRY;
4414
4415   /* add to the auto-address list */
4416   if (twice_nat)
4417     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4418   else
4419     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4420
4421   /* If the address is already bound - or static - add it now */
4422   if (first_int_addr)
4423     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4424
4425   return 0;
4426 }
4427
4428 int
4429 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4430                    snat_protocol_t proto, u32 vrf_id, int is_in)
4431 {
4432   snat_main_per_thread_data_t *tsm;
4433   clib_bihash_kv_8_8_t kv, value;
4434   ip4_header_t ip;
4435   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4436   snat_session_key_t key;
4437   snat_session_t *s;
4438   clib_bihash_8_8_t *t;
4439
4440   if (sm->endpoint_dependent)
4441     return VNET_API_ERROR_UNSUPPORTED;
4442
4443   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4444   if (sm->num_workers > 1)
4445     tsm =
4446       vec_elt_at_index (sm->per_thread_data,
4447                         sm->worker_in2out_cb (&ip, fib_index, 0));
4448   else
4449     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4450
4451   key.addr.as_u32 = addr->as_u32;
4452   key.port = clib_host_to_net_u16 (port);
4453   key.protocol = proto;
4454   key.fib_index = fib_index;
4455   kv.key = key.as_u64;
4456   t = is_in ? &tsm->in2out : &tsm->out2in;
4457   if (!clib_bihash_search_8_8 (t, &kv, &value))
4458     {
4459       if (pool_is_free_index (tsm->sessions, value.value))
4460         return VNET_API_ERROR_UNSPECIFIED;
4461
4462       s = pool_elt_at_index (tsm->sessions, value.value);
4463       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4464       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4465       return 0;
4466     }
4467
4468   return VNET_API_ERROR_NO_SUCH_ENTRY;
4469 }
4470
4471 int
4472 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4473                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4474                       u32 vrf_id, int is_in)
4475 {
4476   ip4_header_t ip;
4477   clib_bihash_16_8_t *t;
4478   clib_bihash_kv_16_8_t kv, value;
4479   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4480   snat_session_t *s;
4481   snat_main_per_thread_data_t *tsm;
4482
4483   if (!sm->endpoint_dependent)
4484     return VNET_API_ERROR_FEATURE_DISABLED;
4485
4486   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4487   if (sm->num_workers > 1)
4488     tsm =
4489       vec_elt_at_index (sm->per_thread_data,
4490                         sm->worker_in2out_cb (&ip, fib_index, 0));
4491   else
4492     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4493
4494   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4495   make_ed_kv (addr, eh_addr, proto, fib_index, clib_host_to_net_u16 (port),
4496               clib_host_to_net_u16 (eh_port), ~0ULL, &kv);
4497   if (clib_bihash_search_16_8 (t, &kv, &value))
4498     {
4499       return VNET_API_ERROR_NO_SUCH_ENTRY;
4500     }
4501
4502   if (pool_is_free_index (tsm->sessions, value.value))
4503     return VNET_API_ERROR_UNSPECIFIED;
4504   s = pool_elt_at_index (tsm->sessions, value.value);
4505   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4506   nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
4507   return 0;
4508 }
4509
4510 void
4511 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4512 {
4513   snat_main_t *sm = &snat_main;
4514
4515   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4516   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4517   sm->psid = psid;
4518   sm->psid_offset = psid_offset;
4519   sm->psid_length = psid_length;
4520 }
4521
4522 void
4523 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4524 {
4525   snat_main_t *sm = &snat_main;
4526
4527   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4528   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4529   sm->start_port = start_port;
4530   sm->end_port = end_port;
4531 }
4532
4533 void
4534 nat_set_alloc_addr_and_port_default (void)
4535 {
4536   snat_main_t *sm = &snat_main;
4537
4538   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4539   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4540 }
4541
4542 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4543                                  vlib_node_runtime_t * node,
4544                                  vlib_frame_t * frame)
4545 {
4546   return 0;
4547 }
4548
4549 /* *INDENT-OFF* */
4550 VLIB_REGISTER_NODE (nat_default_node) = {
4551   .name = "nat-default",
4552   .vector_size = sizeof (u32),
4553   .format_trace = 0,
4554   .type = VLIB_NODE_TYPE_INTERNAL,
4555   .n_errors = 0,
4556   .n_next_nodes = NAT_N_NEXT,
4557   .next_nodes = {
4558     [NAT_NEXT_DROP] = "error-drop",
4559     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4560     //[NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4561     //[NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4562     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4563     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4564     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4565     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4566     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4567     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4568     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4569   },
4570 };
4571 /* *INDENT-ON* */
4572
4573 /*
4574  * fd.io coding-style-patch-verification: ON
4575  *
4576  * Local Variables:
4577  * eval: (c-set-style "gnu")
4578  * End:
4579  */