src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36 #include <vppinfra/bihash_16_8.h>
  37 #include <nat/nat44/ed_inlines.h>
  38
  39 #include <vpp/app/version.h>
  40
  41 snat_main_t snat_main;
  42
  43 fib_source_t nat_fib_src_hi;
  44 fib_source_t nat_fib_src_low;
  45
  46 /* *INDENT-OFF* */
  47 /* Hook up input features */
  48 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat-pre-in2out",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-sv-reassembly-feature"),
  53 };
  54 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat-pre-out2in",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  58                                "ip4-dhcp-client-detect",
  59                                "ip4-sv-reassembly-feature"),
  60 };
  61 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  62   .arc_name = "ip4-unicast",
  63   .node_name = "nat44-in2out-worker-handoff",
  64   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  65 };
  66 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  67   .arc_name = "ip4-unicast",
  68   .node_name = "nat44-out2in-worker-handoff",
  69   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  70                                "ip4-dhcp-client-detect"),
  71 };
  72 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  73   .arc_name = "ip4-unicast",
  74   .node_name = "nat44-in2out",
  75   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  76 };
  77 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  78   .arc_name = "ip4-unicast",
  79   .node_name = "nat44-out2in",
  80   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  81                                "ip4-dhcp-client-detect"),
  82 };
  83 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-det-in2out",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  92 };
  93 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  94   .arc_name = "ip4-unicast",
  95   .node_name = "nat44-det-out2in",
  96   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  97                                "ip4-dhcp-client-detect"),
  98 };
  99 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
 100   .arc_name = "ip4-unicast",
 101   .node_name = "nat44-det-classify",
 102   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-ed-in2out",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 108 };
 109 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-ed-out2in",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-classify",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-handoff-classify",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 124 };
 125 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 126   .arc_name = "ip4-unicast",
 127   .node_name = "nat44-in2out-fast",
 128   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 129 };
 130 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 131   .arc_name = "ip4-unicast",
 132   .node_name = "nat44-out2in-fast",
 133   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 134                                "ip4-dhcp-client-detect"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 137   .arc_name = "ip4-unicast",
 138   .node_name = "nat44-hairpin-dst",
 139   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 140 };
 141 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 142   .arc_name = "ip4-unicast",
 143   .node_name = "nat44-ed-hairpin-dst",
 144   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 145 };
 146
 147 /* Hook up output features */
 148 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-in2out-output",
 151   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 152 };
 153 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 154   .arc_name = "ip4-output",
 155   .node_name = "nat44-in2out-output-worker-handoff",
 156   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 157 };
 158 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 159   .arc_name = "ip4-output",
 160   .node_name = "nat44-hairpin-src",
 161   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 164   .arc_name = "ip4-output",
 165   .node_name = "nat44-ed-in2out-output",
 166   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 167   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 168 };
 169 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 170   .arc_name = "ip4-output",
 171   .node_name = "nat44-ed-hairpin-src",
 172   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 173   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 174 };
 175
 176 /* Hook up ip4-local features */
 177 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 178 {
 179   .arc_name = "ip4-local",
 180   .node_name = "nat44-hairpinning",
 181   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 182 };
 183 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 184 {
 185   .arc_name = "ip4-local",
 186   .node_name = "nat44-ed-hairpinning",
 187   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 188 };
 189
 190
 191 VLIB_PLUGIN_REGISTER () = {
 192     .version = VPP_BUILD_VER,
 193     .description = "Network Address Translation (NAT)",
 194 };
 195 /* *INDENT-ON* */
 196
 197 void
 198 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 199                        u8 is_ha)
 200 {
 201   snat_session_key_t key;
 202   clib_bihash_kv_8_8_t kv;
 203   u8 proto;
 204   u16 r_port, l_port;
 205   ip4_address_t *l_addr, *r_addr;
 206   u32 fib_index = 0;
 207   clib_bihash_kv_16_8_t ed_kv;
 208   snat_main_per_thread_data_t *tsm =
 209     vec_elt_at_index (sm->per_thread_data, thread_index);
 210
 211   if (is_fwd_bypass_session (s))
 212     {
 213       if (snat_is_unk_proto_session (s))
 214         {
 215           make_ed_kv (&s->in2out.addr, &s->ext_host_addr, s->in2out.port, 0,
 216                       0, 0, ~0, ~0, &ed_kv);
 217         }
 218       else
 219         {
 220           l_port = s->in2out.port;
 221           r_port = s->ext_host_port;
 222           l_addr = &s->in2out.addr;
 223           r_addr = &s->ext_host_addr;
 224           proto = nat_proto_to_ip_proto (s->in2out.protocol);
 225           make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0,
 226                       ~0, &ed_kv);
 227         }
 228       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 229         nat_elog_warn ("in2out_ed key del failed");
 230       return;
 231     }
 232
 233   /* session lookup tables */
 234   if (is_ed_session (s))
 235     {
 236       if (is_affinity_sessions (s))
 237         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 238                              s->in2out.protocol, s->out2in.port);
 239       l_addr = &s->out2in.addr;
 240       r_addr = &s->ext_host_addr;
 241       fib_index = s->out2in.fib_index;
 242       if (snat_is_unk_proto_session (s))
 243         {
 244           proto = s->in2out.port;
 245           r_port = 0;
 246           l_port = 0;
 247         }
 248       else
 249         {
 250           proto = nat_proto_to_ip_proto (s->in2out.protocol);
 251           l_port = s->out2in.port;
 252           r_port = s->ext_host_port;
 253         }
 254       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0, ~0,
 255                   &ed_kv);
 256       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 257         nat_elog_warn ("out2in_ed key del failed");
 258       l_addr = &s->in2out.addr;
 259       fib_index = s->in2out.fib_index;
 260       if (!snat_is_unk_proto_session (s))
 261         l_port = s->in2out.port;
 262       if (is_twice_nat_session (s))
 263         {
 264           r_addr = &s->ext_host_nat_addr;
 265           r_port = s->ext_host_nat_port;
 266         }
 267       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0, ~0,
 268                   &ed_kv);
 269       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 270         nat_elog_warn ("in2out_ed key del failed");
 271
 272       if (!is_ha)
 273         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 274                                &s->in2out.addr, s->in2out.port,
 275                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 276                                &s->out2in.addr, s->out2in.port,
 277                                &s->ext_host_addr, s->ext_host_port,
 278                                s->in2out.protocol, is_twice_nat_session (s));
 279     }
 280   else
 281     {
 282       kv.key = s->in2out.as_u64;
 283       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 284         nat_elog_warn ("in2out key del failed");
 285       kv.key = s->out2in.as_u64;
 286       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 287         nat_elog_warn ("out2in key del failed");
 288
 289       if (!is_ha)
 290         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 291                                  &s->in2out.addr, s->in2out.port,
 292                                  &s->out2in.addr, s->out2in.port,
 293                                  s->in2out.protocol);
 294     }
 295
 296   if (snat_is_unk_proto_session (s))
 297     return;
 298
 299   if (!is_ha)
 300     {
 301       /* log NAT event */
 302       snat_ipfix_logging_nat44_ses_delete (thread_index,
 303                                            s->in2out.addr.as_u32,
 304                                            s->out2in.addr.as_u32,
 305                                            s->in2out.protocol,
 306                                            s->in2out.port,
 307                                            s->out2in.port,
 308                                            s->in2out.fib_index);
 309
 310       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 311                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 312                    thread_index);
 313     }
 314
 315   /* Twice NAT address and port for external host */
 316   if (is_twice_nat_session (s))
 317     {
 318       key.protocol = s->in2out.protocol;
 319       key.port = s->ext_host_nat_port;
 320       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 321       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 322                                           thread_index, &key);
 323     }
 324
 325   if (snat_is_session_static (s))
 326     return;
 327
 328   snat_free_outside_address_and_port (sm->addresses, thread_index,
 329                                       &s->out2in);
 330 }
 331
 332 int
 333 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
 334 {
 335   snat_main_t *sm = &snat_main;
 336   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
 337   u32 len = vec_len (sm->max_translations_per_fib);
 338
 339   if (len <= fib_index)
 340     {
 341       vec_validate (sm->max_translations_per_fib, fib_index + 1);
 342
 343       for (; len < vec_len (sm->max_translations_per_fib); len++)
 344         sm->max_translations_per_fib[len] = sm->max_translations;
 345     }
 346
 347   sm->max_translations_per_fib[fib_index] = session_limit;
 348   return 0;
 349 }
 350
 351 void
 352 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 353                          u32 thread_index, u8 is_ha)
 354 {
 355   snat_session_key_t key;
 356   u8 proto;
 357   u16 r_port, l_port;
 358   ip4_address_t *l_addr, *r_addr;
 359   u32 fib_index;
 360   clib_bihash_kv_16_8_t ed_kv;
 361   snat_main_per_thread_data_t *tsm =
 362     vec_elt_at_index (sm->per_thread_data, thread_index);
 363
 364   if (is_fwd_bypass_session (s))
 365     {
 366       if (snat_is_unk_proto_session (s))
 367         {
 368           proto = s->in2out.port;
 369           r_port = 0;
 370           l_port = 0;
 371         }
 372       else
 373         {
 374           proto = nat_proto_to_ip_proto (s->in2out.protocol);
 375           l_port = s->in2out.port;
 376           r_port = s->ext_host_port;
 377         }
 378
 379       l_addr = &s->in2out.addr;
 380       r_addr = &s->ext_host_addr;
 381       fib_index = 0;
 382       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0, ~0,
 383                   &ed_kv);
 384
 385       if (PREDICT_FALSE
 386           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 387         nat_elog_warn ("in2out_ed key del failed");
 388       return;
 389     }
 390
 391   /* session lookup tables */
 392   if (is_affinity_sessions (s))
 393     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 394                          s->in2out.protocol, s->out2in.port);
 395   l_addr = &s->out2in.addr;
 396   r_addr = &s->ext_host_addr;
 397   fib_index = s->out2in.fib_index;
 398   if (snat_is_unk_proto_session (s))
 399     {
 400       proto = s->in2out.port;
 401       r_port = 0;
 402       l_port = 0;
 403     }
 404   else
 405     {
 406       proto = nat_proto_to_ip_proto (s->in2out.protocol);
 407       l_port = s->out2in.port;
 408       r_port = s->ext_host_port;
 409     }
 410   make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0, ~0,
 411               &ed_kv);
 412
 413   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0)))
 414     nat_elog_warn ("out2in_ed key del failed");
 415
 416   l_addr = &s->in2out.addr;
 417   fib_index = s->in2out.fib_index;
 418
 419   if (!snat_is_unk_proto_session (s))
 420     l_port = s->in2out.port;
 421
 422   if (is_twice_nat_session (s))
 423     {
 424       r_addr = &s->ext_host_nat_addr;
 425       r_port = s->ext_host_nat_port;
 426     }
 427   make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0, ~0,
 428               &ed_kv);
 429
 430   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 431     nat_elog_warn ("in2out_ed key del failed");
 432
 433   if (!is_ha)
 434     {
 435       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 436                              &s->in2out.addr, s->in2out.port,
 437                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 438                              &s->out2in.addr, s->out2in.port,
 439                              &s->ext_host_addr, s->ext_host_port,
 440                              s->in2out.protocol, is_twice_nat_session (s));
 441     }
 442
 443   if (snat_is_unk_proto_session (s))
 444     return;
 445
 446   if (!is_ha)
 447     {
 448       snat_ipfix_logging_nat44_ses_delete (thread_index,
 449                                            s->in2out.addr.as_u32,
 450                                            s->out2in.addr.as_u32,
 451                                            s->in2out.protocol,
 452                                            s->in2out.port,
 453                                            s->out2in.port,
 454                                            s->in2out.fib_index);
 455       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 456                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 457                    thread_index);
 458     }
 459
 460   /* Twice NAT address and port for external host */
 461   if (is_twice_nat_session (s))
 462     {
 463       key.protocol = s->in2out.protocol;
 464       key.port = s->ext_host_nat_port;
 465       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 466       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 467                                           thread_index, &key);
 468     }
 469
 470   if (snat_is_session_static (s))
 471     return;
 472
 473   snat_free_outside_address_and_port (sm->addresses, thread_index,
 474                                       &s->out2in);
 475 }
 476
 477
 478 snat_user_t *
 479 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 480                         u32 thread_index)
 481 {
 482   snat_user_t *u = 0;
 483   snat_user_key_t user_key;
 484   clib_bihash_kv_8_8_t kv, value;
 485   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 486   dlist_elt_t *per_user_list_head_elt;
 487
 488   user_key.addr.as_u32 = addr->as_u32;
 489   user_key.fib_index = fib_index;
 490   kv.key = user_key.as_u64;
 491
 492   /* Ever heard of the "user" = src ip4 address before? */
 493   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 494     {
 495       /* no, make a new one */
 496       pool_get (tsm->users, u);
 497       clib_memset (u, 0, sizeof (*u));
 498
 499       u->addr.as_u32 = addr->as_u32;
 500       u->fib_index = fib_index;
 501
 502       pool_get (tsm->list_pool, per_user_list_head_elt);
 503
 504       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 505         tsm->list_pool;
 506
 507       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 508
 509       kv.value = u - tsm->users;
 510
 511       /* add user */
 512       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 513         {
 514           nat_elog_warn ("user_hash key add failed");
 515           nat44_delete_user_with_no_session (sm, u, thread_index);
 516           return NULL;
 517         }
 518
 519       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 520                                pool_elts (tsm->users));
 521     }
 522   else
 523     {
 524       u = pool_elt_at_index (tsm->users, value.value);
 525     }
 526
 527   return u;
 528 }
 529
 530 snat_session_t *
 531 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 532                               u32 thread_index, f64 now)
 533 {
 534   snat_session_t *s;
 535   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 536   u32 oldest_per_user_translation_list_index, session_index;
 537   dlist_elt_t *oldest_per_user_translation_list_elt;
 538   dlist_elt_t *per_user_translation_list_elt;
 539
 540   /* Over quota? Recycle the least recently used translation */
 541   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 542     {
 543       oldest_per_user_translation_list_index =
 544         clib_dlist_remove_head (tsm->list_pool,
 545                                 u->sessions_per_user_list_head_index);
 546
 547       ASSERT (oldest_per_user_translation_list_index != ~0);
 548
 549       /* Add it back to the end of the LRU list */
 550       clib_dlist_addtail (tsm->list_pool,
 551                           u->sessions_per_user_list_head_index,
 552                           oldest_per_user_translation_list_index);
 553       /* Get the list element */
 554       oldest_per_user_translation_list_elt =
 555         pool_elt_at_index (tsm->list_pool,
 556                            oldest_per_user_translation_list_index);
 557
 558       /* Get the session index from the list element */
 559       session_index = oldest_per_user_translation_list_elt->value;
 560
 561       /* Get the session */
 562       s = pool_elt_at_index (tsm->sessions, session_index);
 563       nat_free_session_data (sm, s, thread_index, 0);
 564       if (snat_is_session_static (s))
 565         u->nstaticsessions--;
 566       else
 567         u->nsessions--;
 568       s->flags = 0;
 569       s->total_bytes = 0;
 570       s->total_pkts = 0;
 571       s->state = 0;
 572       s->ext_host_addr.as_u32 = 0;
 573       s->ext_host_port = 0;
 574       s->ext_host_nat_addr.as_u32 = 0;
 575       s->ext_host_nat_port = 0;
 576     }
 577   else
 578     {
 579       pool_get (tsm->sessions, s);
 580       clib_memset (s, 0, sizeof (*s));
 581
 582       /* Create list elts */
 583       pool_get (tsm->list_pool, per_user_translation_list_elt);
 584       clib_dlist_init (tsm->list_pool,
 585                        per_user_translation_list_elt - tsm->list_pool);
 586
 587       per_user_translation_list_elt->value = s - tsm->sessions;
 588       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 589       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 590
 591       clib_dlist_addtail (tsm->list_pool,
 592                           s->per_user_list_head_index,
 593                           per_user_translation_list_elt - tsm->list_pool);
 594
 595       s->user_index = u - tsm->users;
 596       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 597                                pool_elts (tsm->sessions));
 598     }
 599
 600   s->ha_last_refreshed = now;
 601
 602   return s;
 603 }
 604
 605 void
 606 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 607                           int is_add)
 608 {
 609   fib_prefix_t prefix = {
 610     .fp_len = p_len,
 611     .fp_proto = FIB_PROTOCOL_IP4,
 612     .fp_addr = {
 613                 .ip4.as_u32 = addr->as_u32,
 614                 },
 615   };
 616   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 617
 618   if (is_add)
 619     fib_table_entry_update_one_path (fib_index,
 620                                      &prefix,
 621                                      nat_fib_src_low,
 622                                      (FIB_ENTRY_FLAG_CONNECTED |
 623                                       FIB_ENTRY_FLAG_LOCAL |
 624                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 625                                      DPO_PROTO_IP4,
 626                                      NULL,
 627                                      sw_if_index,
 628                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 629   else
 630     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 631 }
 632
 633 int
 634 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 635                   u8 twice_nat)
 636 {
 637   snat_address_t *ap;
 638   snat_interface_t *i;
 639   vlib_thread_main_t *tm = vlib_get_thread_main ();
 640
 641   if (twice_nat && !sm->endpoint_dependent)
 642     return VNET_API_ERROR_FEATURE_DISABLED;
 643
 644   /* Check if address already exists */
 645   /* *INDENT-OFF* */
 646   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 647     {
 648       if (ap->addr.as_u32 == addr->as_u32)
 649         return VNET_API_ERROR_VALUE_EXIST;
 650     }
 651   /* *INDENT-ON* */
 652
 653   if (twice_nat)
 654     vec_add2 (sm->twice_nat_addresses, ap, 1);
 655   else
 656     vec_add2 (sm->addresses, ap, 1);
 657
 658   ap->addr = *addr;
 659   if (vrf_id != ~0)
 660     ap->fib_index =
 661       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 662                                          nat_fib_src_low);
 663   else
 664     ap->fib_index = ~0;
 665 #define _(N, i, n, s) \
 666   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 667   ap->busy_##n##_ports = 0; \
 668   ap->busy_##n##_ports_per_thread = 0;\
 669   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 670   foreach_nat_protocol
 671 #undef _
 672     if (twice_nat)
 673     return 0;
 674
 675   /* Add external address to FIB */
 676   /* *INDENT-OFF* */
 677   pool_foreach (i, sm->interfaces,
 678   ({
 679     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 680       continue;
 681
 682     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 683     break;
 684   }));
 685   pool_foreach (i, sm->output_feature_interfaces,
 686   ({
 687     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 688       continue;
 689
 690     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 691     break;
 692   }));
 693   /* *INDENT-ON* */
 694
 695   return 0;
 696 }
 697
 698 static int
 699 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 700 {
 701   snat_static_mapping_t *m;
 702   /* *INDENT-OFF* */
 703   pool_foreach (m, sm->static_mappings,
 704   ({
 705       if (is_addr_only_static_mapping (m) ||
 706           is_out2in_only_static_mapping (m) ||
 707           is_identity_static_mapping (m))
 708         continue;
 709       if (m->external_addr.as_u32 == addr.as_u32)
 710         return 1;
 711   }));
 712   /* *INDENT-ON* */
 713
 714   return 0;
 715 }
 716
 717 static void
 718 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 719                                        ip4_address_t l_addr,
 720                                        u16 l_port,
 721                                        u32 sw_if_index,
 722                                        u16 e_port,
 723                                        u32 vrf_id,
 724                                        nat_protocol_t proto,
 725                                        int addr_only, int is_add, u8 * tag,
 726                                        int twice_nat, int out2in_only,
 727                                        int identity_nat)
 728 {
 729   snat_static_map_resolve_t *rp;
 730
 731   vec_add2 (sm->to_resolve, rp, 1);
 732   rp->l_addr.as_u32 = l_addr.as_u32;
 733   rp->l_port = l_port;
 734   rp->sw_if_index = sw_if_index;
 735   rp->e_port = e_port;
 736   rp->vrf_id = vrf_id;
 737   rp->proto = proto;
 738   rp->addr_only = addr_only;
 739   rp->is_add = is_add;
 740   rp->twice_nat = twice_nat;
 741   rp->out2in_only = out2in_only;
 742   rp->identity_nat = identity_nat;
 743   rp->tag = vec_dup (tag);
 744 }
 745
 746 static u32
 747 get_thread_idx_by_port (u16 e_port)
 748 {
 749   snat_main_t *sm = &snat_main;
 750   u32 thread_idx = sm->num_workers;
 751   if (sm->num_workers > 1)
 752     {
 753       thread_idx =
 754         sm->first_worker_index +
 755         sm->workers[(e_port - 1024) / sm->port_per_thread];
 756     }
 757   return thread_idx;
 758 }
 759
 760 void
 761 snat_static_mapping_del_sessions (snat_main_t * sm,
 762                                   snat_main_per_thread_data_t * tsm,
 763                                   snat_user_key_t u_key, int addr_only,
 764                                   ip4_address_t e_addr, u16 e_port)
 765 {
 766   clib_bihash_kv_8_8_t kv, value;
 767   kv.key = u_key.as_u64;
 768   u64 user_index;
 769   dlist_elt_t *head, *elt;
 770   snat_user_t *u;
 771   snat_session_t *s;
 772   u32 elt_index, head_index, ses_index;
 773   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 774     {
 775       user_index = value.value;
 776       u = pool_elt_at_index (tsm->users, user_index);
 777       if (u->nstaticsessions)
 778         {
 779           head_index = u->sessions_per_user_list_head_index;
 780           head = pool_elt_at_index (tsm->list_pool, head_index);
 781           elt_index = head->next;
 782           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 783           ses_index = elt->value;
 784           while (ses_index != ~0)
 785             {
 786               s = pool_elt_at_index (tsm->sessions, ses_index);
 787               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 788               ses_index = elt->value;
 789
 790               if (!addr_only)
 791                 {
 792                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 793                       (clib_net_to_host_u16 (s->out2in.port) != e_port))
 794                     continue;
 795                 }
 796
 797               if (is_lb_session (s))
 798                 continue;
 799
 800               if (!snat_is_session_static (s))
 801                 continue;
 802
 803               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 804               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 805
 806               if (!addr_only)
 807                 break;
 808             }
 809         }
 810     }
 811 }
 812
 813 void
 814 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 815                                      snat_main_per_thread_data_t * tsm,
 816                                      ip4_address_t l_addr,
 817                                      u16 l_port,
 818                                      u8 protocol,
 819                                      u32 fib_index, int addr_only,
 820                                      ip4_address_t e_addr, u16 e_port)
 821 {
 822   snat_session_t *s;
 823   u32 *indexes_to_free = NULL;
 824   /* *INDENT-OFF* */
 825   pool_foreach (s, tsm->sessions, {
 826     if (s->in2out.fib_index != fib_index ||
 827         s->in2out.addr.as_u32 != l_addr.as_u32)
 828       {
 829         continue;
 830       }
 831     if (!addr_only)
 832       {
 833         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 834             (clib_net_to_host_u16 (s->out2in.port) != e_port) ||
 835             clib_net_to_host_u16 (s->in2out.port) != l_port ||
 836             s->in2out.protocol != protocol)
 837           continue;
 838       }
 839
 840     if (is_lb_session (s))
 841       continue;
 842     if (!snat_is_session_static (s))
 843       continue;
 844     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 845     vec_add1 (indexes_to_free, s - tsm->sessions);
 846     if (!addr_only)
 847       break;
 848   });
 849   /* *INDENT-ON* */
 850   u32 *ses_index;
 851   vec_foreach (ses_index, indexes_to_free)
 852   {
 853     s = pool_elt_at_index (tsm->sessions, *ses_index);
 854     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 855   }
 856   vec_free (indexes_to_free);
 857 }
 858
 859 int
 860 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 861                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 862                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 863                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 864                          u8 identity_nat)
 865 {
 866   snat_main_t *sm = &snat_main;
 867   snat_static_mapping_t *m;
 868   snat_session_key_t m_key;
 869   clib_bihash_kv_8_8_t kv, value;
 870   snat_address_t *a = 0;
 871   u32 fib_index = ~0;
 872   snat_interface_t *interface;
 873   int i;
 874   snat_main_per_thread_data_t *tsm;
 875   snat_user_key_t u_key;
 876   snat_user_t *u;
 877   dlist_elt_t *head, *elt;
 878   u32 elt_index, head_index;
 879   u32 ses_index;
 880   u64 user_index;
 881   snat_session_t *s;
 882   snat_static_map_resolve_t *rp, *rp_match = 0;
 883   nat44_lb_addr_port_t *local;
 884   u32 find = ~0;
 885
 886   if (!sm->endpoint_dependent)
 887     {
 888       if (twice_nat || out2in_only)
 889         return VNET_API_ERROR_FEATURE_DISABLED;
 890     }
 891
 892   /* If the external address is a specific interface address */
 893   if (sw_if_index != ~0)
 894     {
 895       ip4_address_t *first_int_addr;
 896
 897       for (i = 0; i < vec_len (sm->to_resolve); i++)
 898         {
 899           rp = sm->to_resolve + i;
 900           if (rp->sw_if_index != sw_if_index ||
 901               rp->l_addr.as_u32 != l_addr.as_u32 ||
 902               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 903             continue;
 904
 905           if (!addr_only)
 906             {
 907               if ((rp->l_port != l_port && rp->e_port != e_port)
 908                   || rp->proto != proto)
 909                 continue;
 910             }
 911
 912           rp_match = rp;
 913           break;
 914         }
 915
 916       /* Might be already set... */
 917       first_int_addr = ip4_interface_first_address
 918         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 919
 920       if (is_add)
 921         {
 922           if (rp_match)
 923             return VNET_API_ERROR_VALUE_EXIST;
 924
 925           snat_add_static_mapping_when_resolved
 926             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 927              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 928
 929           /* DHCP resolution required? */
 930           if (first_int_addr == 0)
 931             {
 932               return 0;
 933             }
 934           else
 935             {
 936               e_addr.as_u32 = first_int_addr->as_u32;
 937               /* Identity mapping? */
 938               if (l_addr.as_u32 == 0)
 939                 l_addr.as_u32 = e_addr.as_u32;
 940             }
 941         }
 942       else
 943         {
 944           if (!rp_match)
 945             return VNET_API_ERROR_NO_SUCH_ENTRY;
 946
 947           vec_del1 (sm->to_resolve, i);
 948
 949           if (first_int_addr)
 950             {
 951               e_addr.as_u32 = first_int_addr->as_u32;
 952               /* Identity mapping? */
 953               if (l_addr.as_u32 == 0)
 954                 l_addr.as_u32 = e_addr.as_u32;
 955             }
 956           else
 957             return 0;
 958         }
 959     }
 960
 961   m_key.addr = e_addr;
 962   m_key.port = addr_only ? 0 : e_port;
 963   m_key.protocol = addr_only ? 0 : proto;
 964   m_key.fib_index = 0;
 965   kv.key = m_key.as_u64;
 966   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 967     m = 0;
 968   else
 969     m = pool_elt_at_index (sm->static_mappings, value.value);
 970
 971   if (is_add)
 972     {
 973       if (m)
 974         {
 975           if (is_identity_static_mapping (m))
 976             {
 977               /* *INDENT-OFF* */
 978               pool_foreach (local, m->locals,
 979               ({
 980                 if (local->vrf_id == vrf_id)
 981                   return VNET_API_ERROR_VALUE_EXIST;
 982               }));
 983               /* *INDENT-ON* */
 984               pool_get (m->locals, local);
 985               local->vrf_id = vrf_id;
 986               local->fib_index =
 987                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 988                                                    nat_fib_src_low);
 989               m_key.addr = m->local_addr;
 990               m_key.port = m->local_port;
 991               m_key.protocol = m->proto;
 992               m_key.fib_index = local->fib_index;
 993               kv.key = m_key.as_u64;
 994               kv.value = m - sm->static_mappings;
 995               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 996               return 0;
 997             }
 998           else
 999             return VNET_API_ERROR_VALUE_EXIST;
1000         }
1001
1002       if (twice_nat && addr_only)
1003         return VNET_API_ERROR_UNSUPPORTED;
1004
1005       /* Convert VRF id to FIB index */
1006       if (vrf_id != ~0)
1007         fib_index =
1008           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1009                                              nat_fib_src_low);
1010       /* If not specified use inside VRF id from SNAT plugin startup config */
1011       else
1012         {
1013           fib_index = sm->inside_fib_index;
1014           vrf_id = sm->inside_vrf_id;
1015           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1016         }
1017
1018       if (!(out2in_only || identity_nat))
1019         {
1020           m_key.addr = l_addr;
1021           m_key.port = addr_only ? 0 : l_port;
1022           m_key.protocol = addr_only ? 0 : proto;
1023           m_key.fib_index = fib_index;
1024           kv.key = m_key.as_u64;
1025           if (!clib_bihash_search_8_8
1026               (&sm->static_mapping_by_local, &kv, &value))
1027             return VNET_API_ERROR_VALUE_EXIST;
1028         }
1029
1030       /* Find external address in allocated addresses and reserve port for
1031          address and port pair mapping when dynamic translations enabled */
1032       if (!(addr_only || sm->static_mapping_only || out2in_only))
1033         {
1034           for (i = 0; i < vec_len (sm->addresses); i++)
1035             {
1036               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1037                 {
1038                   a = sm->addresses + i;
1039                   /* External port must be unused */
1040                   switch (proto)
1041                     {
1042 #define _(N, j, n, s) \
1043                     case NAT_PROTOCOL_##N: \
1044                       if (a->busy_##n##_port_refcounts[e_port]) \
1045                         return VNET_API_ERROR_INVALID_VALUE; \
1046                       ++a->busy_##n##_port_refcounts[e_port]; \
1047                       if (e_port > 1024) \
1048                         { \
1049                           a->busy_##n##_ports++; \
1050                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1051                         } \
1052                       break;
1053                       foreach_nat_protocol
1054 #undef _
1055                     default:
1056                       nat_elog_info ("unknown protocol");
1057                       return VNET_API_ERROR_INVALID_VALUE_2;
1058                     }
1059                   break;
1060                 }
1061             }
1062           /* External address must be allocated */
1063           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1064             {
1065               if (sw_if_index != ~0)
1066                 {
1067                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1068                     {
1069                       rp = sm->to_resolve + i;
1070                       if (rp->addr_only)
1071                         continue;
1072                       if (rp->sw_if_index != sw_if_index &&
1073                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1074                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1075                           rp->e_port != e_port && rp->proto != proto)
1076                         continue;
1077
1078                       vec_del1 (sm->to_resolve, i);
1079                       break;
1080                     }
1081                 }
1082               return VNET_API_ERROR_NO_SUCH_ENTRY;
1083             }
1084         }
1085
1086       pool_get (sm->static_mappings, m);
1087       clib_memset (m, 0, sizeof (*m));
1088       m->tag = vec_dup (tag);
1089       m->local_addr = l_addr;
1090       m->external_addr = e_addr;
1091       m->twice_nat = twice_nat;
1092       if (out2in_only)
1093         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1094       if (addr_only)
1095         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1096       if (identity_nat)
1097         {
1098           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1099           pool_get (m->locals, local);
1100           local->vrf_id = vrf_id;
1101           local->fib_index = fib_index;
1102         }
1103       else
1104         {
1105           m->vrf_id = vrf_id;
1106           m->fib_index = fib_index;
1107         }
1108       if (!addr_only)
1109         {
1110           m->local_port = l_port;
1111           m->external_port = e_port;
1112           m->proto = proto;
1113         }
1114
1115       if (sm->num_workers > 1)
1116         {
1117           ip4_header_t ip = {
1118             .src_address = m->local_addr,
1119           };
1120           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1121           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1122         }
1123       else
1124         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1125
1126       m_key.addr = m->local_addr;
1127       m_key.port = m->local_port;
1128       m_key.protocol = m->proto;
1129       m_key.fib_index = fib_index;
1130       kv.key = m_key.as_u64;
1131       kv.value = m - sm->static_mappings;
1132       if (!out2in_only)
1133         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1134
1135       m_key.addr = m->external_addr;
1136       m_key.port = m->external_port;
1137       m_key.fib_index = 0;
1138       kv.key = m_key.as_u64;
1139       kv.value = m - sm->static_mappings;
1140       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1141
1142       /* Delete dynamic sessions matching local address (+ local port) */
1143       if (!(sm->static_mapping_only))
1144         {
1145           u_key.addr = m->local_addr;
1146           u_key.fib_index = m->fib_index;
1147           kv.key = u_key.as_u64;
1148           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1149             {
1150               user_index = value.value;
1151               u = pool_elt_at_index (tsm->users, user_index);
1152               if (u->nsessions)
1153                 {
1154                   head_index = u->sessions_per_user_list_head_index;
1155                   head = pool_elt_at_index (tsm->list_pool, head_index);
1156                   elt_index = head->next;
1157                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1158                   ses_index = elt->value;
1159                   while (ses_index != ~0)
1160                     {
1161                       s = pool_elt_at_index (tsm->sessions, ses_index);
1162                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1163                       ses_index = elt->value;
1164
1165                       if (snat_is_session_static (s))
1166                         continue;
1167
1168                       if (!addr_only
1169                           && (clib_net_to_host_u16 (s->in2out.port) !=
1170                               m->local_port))
1171                         continue;
1172
1173                       nat_free_session_data (sm, s,
1174                                              tsm - sm->per_thread_data, 0);
1175                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1176
1177                       if (!addr_only && !sm->endpoint_dependent)
1178                         break;
1179                     }
1180                 }
1181             }
1182         }
1183     }
1184   else
1185     {
1186       if (!m)
1187         {
1188           if (sw_if_index != ~0)
1189             return 0;
1190           else
1191             return VNET_API_ERROR_NO_SUCH_ENTRY;
1192         }
1193
1194       if (identity_nat)
1195         {
1196           if (vrf_id == ~0)
1197             vrf_id = sm->inside_vrf_id;
1198
1199           /* *INDENT-OFF* */
1200           pool_foreach (local, m->locals,
1201           ({
1202             if (local->vrf_id == vrf_id)
1203               find = local - m->locals;
1204           }));
1205           /* *INDENT-ON* */
1206           if (find == ~0)
1207             return VNET_API_ERROR_NO_SUCH_ENTRY;
1208
1209           local = pool_elt_at_index (m->locals, find);
1210           fib_index = local->fib_index;
1211           pool_put (m->locals, local);
1212         }
1213       else
1214         fib_index = m->fib_index;
1215
1216       /* Free external address port */
1217       if (!(addr_only || sm->static_mapping_only || out2in_only))
1218         {
1219           for (i = 0; i < vec_len (sm->addresses); i++)
1220             {
1221               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1222                 {
1223                   a = sm->addresses + i;
1224                   switch (proto)
1225                     {
1226 #define _(N, j, n, s) \
1227                     case NAT_PROTOCOL_##N: \
1228                       --a->busy_##n##_port_refcounts[e_port]; \
1229                       if (e_port > 1024) \
1230                         { \
1231                           a->busy_##n##_ports--; \
1232                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1233                         } \
1234                       break;
1235                       foreach_nat_protocol
1236 #undef _
1237                     default:
1238                       nat_elog_info ("unknown protocol");
1239                       return VNET_API_ERROR_INVALID_VALUE_2;
1240                     }
1241                   break;
1242                 }
1243             }
1244         }
1245
1246       if (sm->num_workers > 1)
1247         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1248       else
1249         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1250
1251       m_key.addr = m->local_addr;
1252       m_key.port = m->local_port;
1253       m_key.protocol = m->proto;
1254       m_key.fib_index = fib_index;
1255       kv.key = m_key.as_u64;
1256       if (!out2in_only)
1257         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1258
1259       /* Delete session(s) for static mapping if exist */
1260       if (!(sm->static_mapping_only) ||
1261           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1262         {
1263           if (sm->endpoint_dependent)
1264             {
1265               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1266                                                    m->local_port, m->proto,
1267                                                    fib_index, addr_only,
1268                                                    e_addr, e_port);
1269             }
1270           else
1271             {
1272               u_key.addr = m->local_addr;
1273               u_key.fib_index = fib_index;
1274               kv.key = u_key.as_u64;
1275               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1276                                                 e_addr, e_port);
1277             }
1278         }
1279
1280       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1281       if (pool_elts (m->locals))
1282         return 0;
1283
1284       m_key.addr = m->external_addr;
1285       m_key.port = m->external_port;
1286       m_key.fib_index = 0;
1287       kv.key = m_key.as_u64;
1288       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1289
1290       vec_free (m->tag);
1291       vec_free (m->workers);
1292       /* Delete static mapping from pool */
1293       pool_put (sm->static_mappings, m);
1294     }
1295
1296   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1297     return 0;
1298
1299   /* Add/delete external address to FIB */
1300   /* *INDENT-OFF* */
1301   pool_foreach (interface, sm->interfaces,
1302   ({
1303     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1304       continue;
1305
1306     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1307     break;
1308   }));
1309   pool_foreach (interface, sm->output_feature_interfaces,
1310   ({
1311     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1312       continue;
1313
1314     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1315     break;
1316   }));
1317   /* *INDENT-ON* */
1318
1319   return 0;
1320 }
1321
1322 int
1323 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1324                                  nat_protocol_t proto,
1325                                  nat44_lb_addr_port_t * locals, u8 is_add,
1326                                  twice_nat_type_t twice_nat, u8 out2in_only,
1327                                  u8 * tag, u32 affinity)
1328 {
1329   snat_main_t *sm = &snat_main;
1330   snat_static_mapping_t *m;
1331   snat_session_key_t m_key;
1332   clib_bihash_kv_8_8_t kv, value;
1333   snat_address_t *a = 0;
1334   int i;
1335   nat44_lb_addr_port_t *local;
1336   snat_main_per_thread_data_t *tsm;
1337   snat_session_t *s;
1338   uword *bitmap = 0;
1339
1340   if (!sm->endpoint_dependent)
1341     return VNET_API_ERROR_FEATURE_DISABLED;
1342
1343   m_key.addr = e_addr;
1344   m_key.port = e_port;
1345   m_key.protocol = proto;
1346   m_key.fib_index = 0;
1347   kv.key = m_key.as_u64;
1348   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1349     m = 0;
1350   else
1351     m = pool_elt_at_index (sm->static_mappings, value.value);
1352
1353   if (is_add)
1354     {
1355       if (m)
1356         return VNET_API_ERROR_VALUE_EXIST;
1357
1358       if (vec_len (locals) < 2)
1359         return VNET_API_ERROR_INVALID_VALUE;
1360
1361       /* Find external address in allocated addresses and reserve port for
1362          address and port pair mapping when dynamic translations enabled */
1363       if (!(sm->static_mapping_only || out2in_only))
1364         {
1365           for (i = 0; i < vec_len (sm->addresses); i++)
1366             {
1367               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1368                 {
1369                   a = sm->addresses + i;
1370                   /* External port must be unused */
1371                   switch (proto)
1372                     {
1373 #define _(N, j, n, s) \
1374                     case NAT_PROTOCOL_##N: \
1375                       if (a->busy_##n##_port_refcounts[e_port]) \
1376                         return VNET_API_ERROR_INVALID_VALUE; \
1377                       ++a->busy_##n##_port_refcounts[e_port]; \
1378                       if (e_port > 1024) \
1379                         { \
1380                           a->busy_##n##_ports++; \
1381                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1382                         } \
1383                       break;
1384                       foreach_nat_protocol
1385 #undef _
1386                     default:
1387                       nat_elog_info ("unknown protocol");
1388                       return VNET_API_ERROR_INVALID_VALUE_2;
1389                     }
1390                   break;
1391                 }
1392             }
1393           /* External address must be allocated */
1394           if (!a)
1395             return VNET_API_ERROR_NO_SUCH_ENTRY;
1396         }
1397
1398       pool_get (sm->static_mappings, m);
1399       clib_memset (m, 0, sizeof (*m));
1400       m->tag = vec_dup (tag);
1401       m->external_addr = e_addr;
1402       m->external_port = e_port;
1403       m->proto = proto;
1404       m->twice_nat = twice_nat;
1405       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1406       if (out2in_only)
1407         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1408       m->affinity = affinity;
1409
1410       if (affinity)
1411         m->affinity_per_service_list_head_index =
1412           nat_affinity_get_per_service_list_head_index ();
1413       else
1414         m->affinity_per_service_list_head_index = ~0;
1415
1416       m_key.addr = m->external_addr;
1417       m_key.port = m->external_port;
1418       m_key.protocol = m->proto;
1419       m_key.fib_index = 0;
1420       kv.key = m_key.as_u64;
1421       kv.value = m - sm->static_mappings;
1422       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1423         {
1424           nat_elog_err ("static_mapping_by_external key add failed");
1425           return VNET_API_ERROR_UNSPECIFIED;
1426         }
1427
1428       m_key.fib_index = m->fib_index;
1429       for (i = 0; i < vec_len (locals); i++)
1430         {
1431           locals[i].fib_index =
1432             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1433                                                locals[i].vrf_id,
1434                                                nat_fib_src_low);
1435           m_key.addr = locals[i].addr;
1436           m_key.fib_index = locals[i].fib_index;
1437           if (!out2in_only)
1438             {
1439               m_key.port = locals[i].port;
1440               kv.key = m_key.as_u64;
1441               kv.value = m - sm->static_mappings;
1442               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1443             }
1444           locals[i].prefix = (i == 0) ? locals[i].probability :
1445             (locals[i - 1].prefix + locals[i].probability);
1446           pool_get (m->locals, local);
1447           *local = locals[i];
1448           if (sm->num_workers > 1)
1449             {
1450               ip4_header_t ip = {
1451                 .src_address = locals[i].addr,
1452               };
1453               bitmap =
1454                 clib_bitmap_set (bitmap,
1455                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1456                                  1);
1457             }
1458         }
1459
1460       /* Assign workers */
1461       if (sm->num_workers > 1)
1462         {
1463           /* *INDENT-OFF* */
1464           clib_bitmap_foreach (i, bitmap,
1465             ({
1466                vec_add1(m->workers, i);
1467             }));
1468           /* *INDENT-ON* */
1469         }
1470     }
1471   else
1472     {
1473       if (!m)
1474         return VNET_API_ERROR_NO_SUCH_ENTRY;
1475
1476       if (!is_lb_static_mapping (m))
1477         return VNET_API_ERROR_INVALID_VALUE;
1478
1479       /* Free external address port */
1480       if (!(sm->static_mapping_only || out2in_only))
1481         {
1482           for (i = 0; i < vec_len (sm->addresses); i++)
1483             {
1484               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1485                 {
1486                   a = sm->addresses + i;
1487                   switch (proto)
1488                     {
1489 #define _(N, j, n, s) \
1490                     case NAT_PROTOCOL_##N: \
1491                       --a->busy_##n##_port_refcounts[e_port]; \
1492                       if (e_port > 1024) \
1493                         { \
1494                           a->busy_##n##_ports--; \
1495                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1496                         } \
1497                       break;
1498                       foreach_nat_protocol
1499 #undef _
1500                     default:
1501                       nat_elog_info ("unknown protocol");
1502                       return VNET_API_ERROR_INVALID_VALUE_2;
1503                     }
1504                   break;
1505                 }
1506             }
1507         }
1508
1509       m_key.addr = m->external_addr;
1510       m_key.port = m->external_port;
1511       m_key.protocol = m->proto;
1512       m_key.fib_index = 0;
1513       kv.key = m_key.as_u64;
1514       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1515         {
1516           nat_elog_err ("static_mapping_by_external key del failed");
1517           return VNET_API_ERROR_UNSPECIFIED;
1518         }
1519
1520       /* *INDENT-OFF* */
1521       pool_foreach (local, m->locals,
1522       ({
1523           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1524                             nat_fib_src_low);
1525           m_key.addr = local->addr;
1526           if (!out2in_only)
1527             {
1528               m_key.port = local->port;
1529               m_key.fib_index = local->fib_index;
1530               kv.key = m_key.as_u64;
1531               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1532                 {
1533                   nat_elog_err ("static_mapping_by_local key del failed");
1534                   return VNET_API_ERROR_UNSPECIFIED;
1535                 }
1536             }
1537
1538           if (sm->num_workers > 1)
1539             {
1540               ip4_header_t ip = {
1541                 .src_address = local->addr,
1542               };
1543               tsm = vec_elt_at_index (sm->per_thread_data,
1544                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1545             }
1546           else
1547             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1548
1549           /* Delete sessions */
1550           pool_foreach (s, tsm->sessions, {
1551             if (!(is_lb_session (s)))
1552               continue;
1553
1554             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1555                 (clib_net_to_host_u16 (s->in2out.port) != local->port))
1556               continue;
1557
1558             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1559             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1560           });
1561       }));
1562       /* *INDENT-ON* */
1563       if (m->affinity)
1564         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1565       pool_free (m->locals);
1566       vec_free (m->tag);
1567       vec_free (m->workers);
1568
1569       pool_put (sm->static_mappings, m);
1570     }
1571
1572   return 0;
1573 }
1574
1575 int
1576 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1577                                        ip4_address_t l_addr, u16 l_port,
1578                                        nat_protocol_t proto, u32 vrf_id,
1579                                        u8 probability, u8 is_add)
1580 {
1581   snat_main_t *sm = &snat_main;
1582   snat_static_mapping_t *m = 0;
1583   snat_session_key_t m_key;
1584   clib_bihash_kv_8_8_t kv, value;
1585   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1586   snat_main_per_thread_data_t *tsm;
1587   snat_session_t *s;
1588   u32 *locals = 0;
1589   uword *bitmap = 0;
1590   int i;
1591
1592   if (!sm->endpoint_dependent)
1593     return VNET_API_ERROR_FEATURE_DISABLED;
1594
1595   m_key.addr = e_addr;
1596   m_key.port = e_port;
1597   m_key.protocol = proto;
1598   m_key.fib_index = 0;
1599   kv.key = m_key.as_u64;
1600   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1601     m = pool_elt_at_index (sm->static_mappings, value.value);
1602
1603   if (!m)
1604     return VNET_API_ERROR_NO_SUCH_ENTRY;
1605
1606   if (!is_lb_static_mapping (m))
1607     return VNET_API_ERROR_INVALID_VALUE;
1608
1609   /* *INDENT-OFF* */
1610   pool_foreach (local, m->locals,
1611   ({
1612     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1613         (local->vrf_id == vrf_id))
1614       {
1615         match_local = local;
1616         break;
1617       }
1618   }));
1619   /* *INDENT-ON* */
1620
1621   if (is_add)
1622     {
1623       if (match_local)
1624         return VNET_API_ERROR_VALUE_EXIST;
1625
1626       pool_get (m->locals, local);
1627       clib_memset (local, 0, sizeof (*local));
1628       local->addr.as_u32 = l_addr.as_u32;
1629       local->port = l_port;
1630       local->probability = probability;
1631       local->vrf_id = vrf_id;
1632       local->fib_index =
1633         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1634                                            nat_fib_src_low);
1635
1636       if (!is_out2in_only_static_mapping (m))
1637         {
1638           m_key.addr = l_addr;
1639           m_key.port = l_port;
1640           m_key.fib_index = local->fib_index;
1641           kv.key = m_key.as_u64;
1642           kv.value = m - sm->static_mappings;
1643           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1644             nat_elog_err ("static_mapping_by_local key add failed");
1645         }
1646     }
1647   else
1648     {
1649       if (!match_local)
1650         return VNET_API_ERROR_NO_SUCH_ENTRY;
1651
1652       if (pool_elts (m->locals) < 3)
1653         return VNET_API_ERROR_UNSPECIFIED;
1654
1655       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1656                         nat_fib_src_low);
1657
1658       if (!is_out2in_only_static_mapping (m))
1659         {
1660           m_key.addr = l_addr;
1661           m_key.port = l_port;
1662           m_key.fib_index = match_local->fib_index;
1663           kv.key = m_key.as_u64;
1664           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1665             nat_elog_err ("static_mapping_by_local key del failed");
1666         }
1667
1668       if (sm->num_workers > 1)
1669         {
1670           ip4_header_t ip = {
1671             .src_address = local->addr,
1672           };
1673           tsm = vec_elt_at_index (sm->per_thread_data,
1674                                   sm->worker_in2out_cb (&ip, m->fib_index,
1675                                                         0));
1676         }
1677       else
1678         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1679
1680       /* Delete sessions */
1681       /* *INDENT-OFF* */
1682       pool_foreach (s, tsm->sessions, {
1683         if (!(is_lb_session (s)))
1684           continue;
1685
1686         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1687             (clib_net_to_host_u16 (s->in2out.port) != match_local->port))
1688           continue;
1689
1690         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1691         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1692       });
1693       /* *INDENT-ON* */
1694
1695       pool_put (m->locals, match_local);
1696     }
1697
1698   vec_free (m->workers);
1699
1700   /* *INDENT-OFF* */
1701   pool_foreach (local, m->locals,
1702   ({
1703     vec_add1 (locals, local - m->locals);
1704     if (sm->num_workers > 1)
1705       {
1706         ip4_header_t ip;
1707         ip.src_address.as_u32 = local->addr.as_u32,
1708         bitmap = clib_bitmap_set (bitmap,
1709                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1710                                   1);
1711       }
1712   }));
1713   /* *INDENT-ON* */
1714
1715   ASSERT (vec_len (locals) > 1);
1716
1717   local = pool_elt_at_index (m->locals, locals[0]);
1718   local->prefix = local->probability;
1719   for (i = 1; i < vec_len (locals); i++)
1720     {
1721       local = pool_elt_at_index (m->locals, locals[i]);
1722       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1723       local->prefix = local->probability + prev_local->prefix;
1724     }
1725
1726   /* Assign workers */
1727   if (sm->num_workers > 1)
1728     {
1729       /* *INDENT-OFF* */
1730       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1731       /* *INDENT-ON* */
1732     }
1733
1734   return 0;
1735 }
1736
1737 int
1738 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1739                   u8 twice_nat)
1740 {
1741   snat_address_t *a = 0;
1742   snat_session_t *ses;
1743   u32 *ses_to_be_removed = 0, *ses_index;
1744   snat_main_per_thread_data_t *tsm;
1745   snat_static_mapping_t *m;
1746   snat_interface_t *interface;
1747   int i;
1748   snat_address_t *addresses =
1749     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1750
1751   /* Find SNAT address */
1752   for (i = 0; i < vec_len (addresses); i++)
1753     {
1754       if (addresses[i].addr.as_u32 == addr.as_u32)
1755         {
1756           a = addresses + i;
1757           break;
1758         }
1759     }
1760   if (!a)
1761     return VNET_API_ERROR_NO_SUCH_ENTRY;
1762
1763   if (delete_sm)
1764     {
1765       /* *INDENT-OFF* */
1766       pool_foreach (m, sm->static_mappings,
1767       ({
1768           if (m->external_addr.as_u32 == addr.as_u32)
1769             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1770                                             m->local_port, m->external_port,
1771                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1772                                             m->proto, 0, m->twice_nat,
1773                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1774       }));
1775       /* *INDENT-ON* */
1776     }
1777   else
1778     {
1779       /* Check if address is used in some static mapping */
1780       if (is_snat_address_used_in_static_mapping (sm, addr))
1781         {
1782           nat_elog_notice ("address used in static mapping");
1783           return VNET_API_ERROR_UNSPECIFIED;
1784         }
1785     }
1786
1787   if (a->fib_index != ~0)
1788     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1789
1790   /* Delete sessions using address */
1791   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1792     {
1793       /* *INDENT-OFF* */
1794       vec_foreach (tsm, sm->per_thread_data)
1795         {
1796           pool_foreach (ses, tsm->sessions, ({
1797             if (ses->out2in.addr.as_u32 == addr.as_u32)
1798               {
1799                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1800                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1801               }
1802           }));
1803
1804           if (sm->endpoint_dependent){
1805               vec_foreach (ses_index, ses_to_be_removed)
1806                 {
1807                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1808                   nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1809                 }
1810           }else{
1811               vec_foreach (ses_index, ses_to_be_removed)
1812                 {
1813                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1814                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1815                 }
1816           }
1817
1818           vec_free (ses_to_be_removed);
1819         }
1820       /* *INDENT-ON* */
1821     }
1822
1823 #define _(N, i, n, s) \
1824   vec_free (a->busy_##n##_ports_per_thread);
1825   foreach_nat_protocol
1826 #undef _
1827     if (twice_nat)
1828     {
1829       vec_del1 (sm->twice_nat_addresses, i);
1830       return 0;
1831     }
1832   else
1833     vec_del1 (sm->addresses, i);
1834
1835   /* Delete external address from FIB */
1836   /* *INDENT-OFF* */
1837   pool_foreach (interface, sm->interfaces,
1838   ({
1839     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1840       continue;
1841
1842     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1843     break;
1844   }));
1845   pool_foreach (interface, sm->output_feature_interfaces,
1846   ({
1847     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1848       continue;
1849
1850     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1851     break;
1852   }));
1853   /* *INDENT-ON* */
1854
1855   return 0;
1856 }
1857
1858 int
1859 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1860 {
1861   snat_main_t *sm = &snat_main;
1862   snat_interface_t *i;
1863   const char *feature_name, *del_feature_name;
1864   snat_address_t *ap;
1865   snat_static_mapping_t *m;
1866   snat_det_map_t *dm;
1867   nat_outside_fib_t *outside_fib;
1868   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1869                                                        sw_if_index);
1870
1871   if (sm->out2in_dpo && !is_inside)
1872     return VNET_API_ERROR_UNSUPPORTED;
1873
1874   /* *INDENT-OFF* */
1875   pool_foreach (i, sm->output_feature_interfaces,
1876   ({
1877     if (i->sw_if_index == sw_if_index)
1878       return VNET_API_ERROR_VALUE_EXIST;
1879   }));
1880   /* *INDENT-ON* */
1881
1882   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1883     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1884   else
1885     {
1886       if (sm->num_workers > 1 && !sm->deterministic)
1887         feature_name =
1888           is_inside ? "nat44-in2out-worker-handoff" :
1889           "nat44-out2in-worker-handoff";
1890       else if (sm->deterministic)
1891         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1892       else if (sm->endpoint_dependent)
1893         {
1894           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1895         }
1896       else
1897         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1898     }
1899
1900   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1901     sm->fq_in2out_index =
1902       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1903
1904   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1905     sm->fq_out2in_index =
1906       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1907
1908   if (!is_inside)
1909     {
1910       /* *INDENT-OFF* */
1911       vec_foreach (outside_fib, sm->outside_fibs)
1912         {
1913           if (outside_fib->fib_index == fib_index)
1914             {
1915               if (is_del)
1916                 {
1917                   outside_fib->refcount--;
1918                   if (!outside_fib->refcount)
1919                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1920                 }
1921               else
1922                 outside_fib->refcount++;
1923               goto feature_set;
1924             }
1925         }
1926       /* *INDENT-ON* */
1927       if (!is_del)
1928         {
1929           vec_add2 (sm->outside_fibs, outside_fib, 1);
1930           outside_fib->refcount = 1;
1931           outside_fib->fib_index = fib_index;
1932         }
1933     }
1934 feature_set:
1935   /* *INDENT-OFF* */
1936   pool_foreach (i, sm->interfaces,
1937   ({
1938     if (i->sw_if_index == sw_if_index)
1939       {
1940         if (is_del)
1941           {
1942             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1943               {
1944                 if (is_inside)
1945                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1946                 else
1947                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1948
1949                 if (sm->num_workers > 1 && !sm->deterministic)
1950                   {
1951                     del_feature_name = "nat44-handoff-classify";
1952                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1953                                                  "nat44-out2in-worker-handoff";
1954                   }
1955                 else if (sm->deterministic)
1956                   {
1957                     del_feature_name = "nat44-det-classify";
1958                     feature_name = !is_inside ?  "nat44-det-in2out" :
1959                                                  "nat44-det-out2in";
1960                   }
1961                 else if (sm->endpoint_dependent)
1962                   {
1963                     del_feature_name = "nat44-ed-classify";
1964                     feature_name = !is_inside ?  "nat-pre-in2out" :
1965                                                  "nat-pre-out2in";
1966                   }
1967                 else
1968                   {
1969                     del_feature_name = "nat44-classify";
1970                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1971                   }
1972
1973                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1974                 if (rv)
1975                   return rv;
1976                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1977                                              sw_if_index, 0, 0, 0);
1978                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1979                                              sw_if_index, 1, 0, 0);
1980                 if (!is_inside)
1981                   {
1982                     if (sm->endpoint_dependent)
1983                       vnet_feature_enable_disable ("ip4-local",
1984                                                    "nat44-ed-hairpinning",
1985                                                    sw_if_index, 1, 0, 0);
1986                     else if (!sm->deterministic)
1987                       vnet_feature_enable_disable ("ip4-local",
1988                                                    "nat44-hairpinning",
1989                                                    sw_if_index, 1, 0, 0);
1990                   }
1991               }
1992             else
1993               {
1994                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1995                 if (rv)
1996                   return rv;
1997                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1998                                              sw_if_index, 0, 0, 0);
1999                 pool_put (sm->interfaces, i);
2000                 if (is_inside)
2001                   {
2002                     if (sm->endpoint_dependent)
2003                       vnet_feature_enable_disable ("ip4-local",
2004                                                    "nat44-ed-hairpinning",
2005                                                    sw_if_index, 0, 0, 0);
2006                     else if (!sm->deterministic)
2007                       vnet_feature_enable_disable ("ip4-local",
2008                                                    "nat44-hairpinning",
2009                                                    sw_if_index, 0, 0, 0);
2010                   }
2011               }
2012           }
2013         else
2014           {
2015             if ((nat_interface_is_inside(i) && is_inside) ||
2016                 (nat_interface_is_outside(i) && !is_inside))
2017               return 0;
2018
2019             if (sm->num_workers > 1 && !sm->deterministic)
2020               {
2021                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2022                                                  "nat44-out2in-worker-handoff";
2023                 feature_name = "nat44-handoff-classify";
2024               }
2025             else if (sm->deterministic)
2026               {
2027                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
2028                                                  "nat44-det-out2in";
2029                 feature_name = "nat44-det-classify";
2030               }
2031             else if (sm->endpoint_dependent)
2032               {
2033                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2034                                                  "nat-pre-out2in";
2035
2036                 feature_name = "nat44-ed-classify";
2037               }
2038             else
2039               {
2040                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2041                 feature_name = "nat44-classify";
2042               }
2043
2044             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2045             if (rv)
2046               return rv;
2047             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2048                                          sw_if_index, 0, 0, 0);
2049             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2050                                          sw_if_index, 1, 0, 0);
2051             if (!is_inside)
2052               {
2053                 if (sm->endpoint_dependent)
2054                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2055                                                sw_if_index, 0, 0, 0);
2056                 else if (!sm->deterministic)
2057                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2058                                                sw_if_index, 0, 0, 0);
2059               }
2060             goto set_flags;
2061           }
2062
2063         goto fib;
2064       }
2065   }));
2066   /* *INDENT-ON* */
2067
2068   if (is_del)
2069     return VNET_API_ERROR_NO_SUCH_ENTRY;
2070
2071   pool_get (sm->interfaces, i);
2072   i->sw_if_index = sw_if_index;
2073   i->flags = 0;
2074   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2075                                0);
2076
2077   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2078   if (rv)
2079     return rv;
2080
2081   if (is_inside && !sm->out2in_dpo)
2082     {
2083       if (sm->endpoint_dependent)
2084         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2085                                      sw_if_index, 1, 0, 0);
2086       else if (!sm->deterministic)
2087         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2088                                      sw_if_index, 1, 0, 0);
2089     }
2090
2091 set_flags:
2092   if (is_inside)
2093     {
2094       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2095       return 0;
2096     }
2097   else
2098     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2099
2100   /* Add/delete external addresses to FIB */
2101 fib:
2102   /* *INDENT-OFF* */
2103   vec_foreach (ap, sm->addresses)
2104     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2105
2106   pool_foreach (m, sm->static_mappings,
2107   ({
2108     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2109       continue;
2110
2111     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2112   }));
2113
2114   pool_foreach (dm, sm->det_maps,
2115   ({
2116     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2117   }));
2118   /* *INDENT-ON* */
2119
2120   return 0;
2121 }
2122
2123 int
2124 snat_interface_add_del_output_feature (u32 sw_if_index,
2125                                        u8 is_inside, int is_del)
2126 {
2127   snat_main_t *sm = &snat_main;
2128   snat_interface_t *i;
2129   snat_address_t *ap;
2130   snat_static_mapping_t *m;
2131   nat_outside_fib_t *outside_fib;
2132   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2133                                                        sw_if_index);
2134
2135
2136   if (sm->deterministic ||
2137       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2138     return VNET_API_ERROR_UNSUPPORTED;
2139
2140   /* *INDENT-OFF* */
2141   pool_foreach (i, sm->interfaces,
2142   ({
2143     if (i->sw_if_index == sw_if_index)
2144       return VNET_API_ERROR_VALUE_EXIST;
2145   }));
2146   /* *INDENT-ON* */
2147
2148   if (!is_inside)
2149     {
2150       /* *INDENT-OFF* */
2151       vec_foreach (outside_fib, sm->outside_fibs)
2152         {
2153           if (outside_fib->fib_index == fib_index)
2154             {
2155               if (is_del)
2156                 {
2157                   outside_fib->refcount--;
2158                   if (!outside_fib->refcount)
2159                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2160                 }
2161               else
2162                 outside_fib->refcount++;
2163               goto feature_set;
2164             }
2165         }
2166       /* *INDENT-ON* */
2167       if (!is_del)
2168         {
2169           vec_add2 (sm->outside_fibs, outside_fib, 1);
2170           outside_fib->refcount = 1;
2171           outside_fib->fib_index = fib_index;
2172         }
2173     }
2174
2175 feature_set:
2176   if (is_inside)
2177     {
2178       if (sm->endpoint_dependent)
2179         {
2180           int rv =
2181             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2182           if (rv)
2183             return rv;
2184           rv =
2185             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2186                                                             !is_del);
2187           if (rv)
2188             return rv;
2189           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2190                                        sw_if_index, !is_del, 0, 0);
2191           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2192                                        sw_if_index, !is_del, 0, 0);
2193         }
2194       else
2195         {
2196           int rv =
2197             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2198           if (rv)
2199             return rv;
2200           rv =
2201             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2202                                                             !is_del);
2203           if (rv)
2204             return rv;
2205           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2206                                        sw_if_index, !is_del, 0, 0);
2207           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2208                                        sw_if_index, !is_del, 0, 0);
2209         }
2210       goto fq;
2211     }
2212
2213   if (sm->num_workers > 1)
2214     {
2215       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2216       if (rv)
2217         return rv;
2218       rv =
2219         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2220       if (rv)
2221         return rv;
2222       vnet_feature_enable_disable ("ip4-unicast",
2223                                    "nat44-out2in-worker-handoff",
2224                                    sw_if_index, !is_del, 0, 0);
2225       vnet_feature_enable_disable ("ip4-output",
2226                                    "nat44-in2out-output-worker-handoff",
2227                                    sw_if_index, !is_del, 0, 0);
2228     }
2229   else
2230     {
2231       if (sm->endpoint_dependent)
2232         {
2233           int rv =
2234             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2235           if (rv)
2236             return rv;
2237           rv =
2238             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2239                                                             !is_del);
2240           if (rv)
2241             return rv;
2242           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2243                                        sw_if_index, !is_del, 0, 0);
2244           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2245                                        sw_if_index, !is_del, 0, 0);
2246         }
2247       else
2248         {
2249           int rv =
2250             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2251           if (rv)
2252             return rv;
2253           rv =
2254             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2255                                                             !is_del);
2256           if (rv)
2257             return rv;
2258           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2259                                        sw_if_index, !is_del, 0, 0);
2260           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2261                                        sw_if_index, !is_del, 0, 0);
2262         }
2263     }
2264
2265 fq:
2266   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2267     sm->fq_in2out_output_index =
2268       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2269
2270   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2271     sm->fq_out2in_index =
2272       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2273
2274   /* *INDENT-OFF* */
2275   pool_foreach (i, sm->output_feature_interfaces,
2276   ({
2277     if (i->sw_if_index == sw_if_index)
2278       {
2279         if (is_del)
2280           pool_put (sm->output_feature_interfaces, i);
2281         else
2282           return VNET_API_ERROR_VALUE_EXIST;
2283
2284         goto fib;
2285       }
2286   }));
2287   /* *INDENT-ON* */
2288
2289   if (is_del)
2290     return VNET_API_ERROR_NO_SUCH_ENTRY;
2291
2292   pool_get (sm->output_feature_interfaces, i);
2293   i->sw_if_index = sw_if_index;
2294   i->flags = 0;
2295   if (is_inside)
2296     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2297   else
2298     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2299
2300   /* Add/delete external addresses to FIB */
2301 fib:
2302   if (is_inside)
2303     return 0;
2304
2305   /* *INDENT-OFF* */
2306   vec_foreach (ap, sm->addresses)
2307     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2308
2309   pool_foreach (m, sm->static_mappings,
2310   ({
2311     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2312       continue;
2313
2314     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2315   }));
2316   /* *INDENT-ON* */
2317
2318   return 0;
2319 }
2320
2321 int
2322 snat_set_workers (uword * bitmap)
2323 {
2324   snat_main_t *sm = &snat_main;
2325   int i, j = 0;
2326
2327   if (sm->num_workers < 2)
2328     return VNET_API_ERROR_FEATURE_DISABLED;
2329
2330   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2331     return VNET_API_ERROR_INVALID_WORKER;
2332
2333   vec_free (sm->workers);
2334   /* *INDENT-OFF* */
2335   clib_bitmap_foreach (i, bitmap,
2336     ({
2337       vec_add1(sm->workers, i);
2338       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2339       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2340       j++;
2341     }));
2342   /* *INDENT-ON* */
2343
2344   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2345
2346   return 0;
2347 }
2348
2349 static void
2350 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2351                          u32 old_fib_index)
2352 {
2353   snat_main_t *sm = &snat_main;
2354   nat_outside_fib_t *outside_fib;
2355   snat_interface_t *i;
2356   u8 is_add = 1;
2357   u8 match = 0;
2358
2359   if (new_fib_index == old_fib_index)
2360     return;
2361
2362   if (!vec_len (sm->outside_fibs))
2363     return;
2364
2365   /* *INDENT-OFF* */
2366   pool_foreach (i, sm->interfaces,
2367     ({
2368       if (i->sw_if_index == sw_if_index)
2369         {
2370           if (!(nat_interface_is_outside (i)))
2371             return;
2372           match = 1;
2373         }
2374     }));
2375
2376   pool_foreach (i, sm->output_feature_interfaces,
2377     ({
2378       if (i->sw_if_index == sw_if_index)
2379         {
2380           if (!(nat_interface_is_outside (i)))
2381             return;
2382           match = 1;
2383         }
2384     }));
2385   /* *INDENT-ON* */
2386
2387   if (!match)
2388     return;
2389
2390   vec_foreach (outside_fib, sm->outside_fibs)
2391   {
2392     if (outside_fib->fib_index == old_fib_index)
2393       {
2394         outside_fib->refcount--;
2395         if (!outside_fib->refcount)
2396           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2397         break;
2398       }
2399   }
2400
2401   vec_foreach (outside_fib, sm->outside_fibs)
2402   {
2403     if (outside_fib->fib_index == new_fib_index)
2404       {
2405         outside_fib->refcount++;
2406         is_add = 0;
2407         break;
2408       }
2409   }
2410
2411   if (is_add)
2412     {
2413       vec_add2 (sm->outside_fibs, outside_fib, 1);
2414       outside_fib->refcount = 1;
2415       outside_fib->fib_index = new_fib_index;
2416     }
2417 }
2418
2419 static void
2420 snat_ip4_table_bind (ip4_main_t * im,
2421                      uword opaque,
2422                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2423 {
2424   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2425 }
2426
2427 static void
2428 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2429                                        uword opaque,
2430                                        u32 sw_if_index,
2431                                        ip4_address_t * address,
2432                                        u32 address_length,
2433                                        u32 if_address_index, u32 is_delete);
2434
2435 static void
2436 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2437                                  uword opaque,
2438                                  u32 sw_if_index,
2439                                  ip4_address_t * address,
2440                                  u32 address_length,
2441                                  u32 if_address_index, u32 is_delete);
2442
2443 static int
2444 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2445                                  u32 fib_index,
2446                                  u32 thread_index,
2447                                  snat_session_key_t * k,
2448                                  u16 port_per_thread, u32 snat_thread_index);
2449
2450 void
2451 test_ed_make_split ()
2452 {
2453   ip4_address_t l_addr;
2454   l_addr.as_u8[0] = 1;
2455   l_addr.as_u8[1] = 1;
2456   l_addr.as_u8[2] = 1;
2457   l_addr.as_u8[3] = 1;
2458   ip4_address_t r_addr;
2459   r_addr.as_u8[0] = 2;
2460   r_addr.as_u8[1] = 2;
2461   r_addr.as_u8[2] = 2;
2462   r_addr.as_u8[3] = 2;
2463   u16 l_port = 40001;
2464   u16 r_port = 40301;
2465   u8 proto = 9;
2466   u32 fib_index = 9000001;
2467   u32 thread_index = 3000000001;
2468   u32 session_index = 3000000221;
2469   clib_bihash_kv_16_8_t kv;
2470   make_ed_kv (&l_addr, &r_addr, proto, fib_index, l_port, r_port,
2471               thread_index, session_index, &kv);
2472   ip4_address_t l_addr2;
2473   ip4_address_t r_addr2;
2474   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2475   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2476   u16 l_port2 = 0;
2477   u16 r_port2 = 0;
2478   u8 proto2 = 0;
2479   u32 fib_index2 = 0;
2480   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2481                &r_port2);
2482   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2483   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2484   ASSERT (l_port == l_port2);
2485   ASSERT (r_port == r_port2);
2486   ASSERT (proto == proto2);
2487   ASSERT (fib_index == fib_index2);
2488   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2489   ASSERT (session_index == ed_value_get_session_index (&kv));
2490 }
2491
2492 static clib_error_t *
2493 snat_init (vlib_main_t * vm)
2494 {
2495   snat_main_t *sm = &snat_main;
2496   clib_error_t *error = 0;
2497   ip4_main_t *im = &ip4_main;
2498   ip_lookup_main_t *lm = &im->lookup_main;
2499   uword *p;
2500   vlib_thread_registration_t *tr;
2501   vlib_thread_main_t *tm = vlib_get_thread_main ();
2502   uword *bitmap = 0;
2503   u32 i;
2504   ip4_add_del_interface_address_callback_t cb4;
2505   vlib_node_t *node;
2506
2507   sm->vnet_main = vnet_get_main ();
2508   sm->ip4_main = im;
2509   sm->ip4_lookup_main = lm;
2510   sm->api_main = vlibapi_get_main ();
2511   sm->first_worker_index = 0;
2512   sm->num_workers = 0;
2513   sm->workers = 0;
2514   sm->port_per_thread = 0xffff - 1024;
2515   sm->fq_in2out_index = ~0;
2516   sm->fq_in2out_output_index = ~0;
2517   sm->fq_out2in_index = ~0;
2518
2519   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2520   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2521   sm->forwarding_enabled = 0;
2522   sm->log_class = vlib_log_register_class ("nat", 0);
2523   sm->log_level = SNAT_LOG_ERROR;
2524   sm->mss_clamping = 0;
2525
2526   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2527   sm->error_node_index = node->index;
2528
2529   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2530   sm->pre_in2out_node_index = node->index;
2531   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2532   sm->pre_out2in_node_index = node->index;
2533
2534   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2535   sm->pre_in2out_node_index = node->index;
2536
2537   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2538   sm->pre_out2in_node_index = node->index;
2539
2540   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2541   sm->in2out_node_index = node->index;
2542   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2543   sm->in2out_output_node_index = node->index;
2544   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2545   sm->in2out_fast_node_index = node->index;
2546   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2547   sm->in2out_slowpath_node_index = node->index;
2548   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2549   sm->in2out_slowpath_output_node_index = node->index;
2550
2551   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2552   sm->ed_in2out_node_index = node->index;
2553   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2554   sm->ed_in2out_slowpath_node_index = node->index;
2555
2556   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2557   sm->out2in_node_index = node->index;
2558   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2559   sm->out2in_fast_node_index = node->index;
2560
2561   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2562   sm->ed_out2in_node_index = node->index;
2563   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2564   sm->ed_out2in_slowpath_node_index = node->index;
2565
2566   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2567   sm->det_in2out_node_index = node->index;
2568   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2569   sm->det_out2in_node_index = node->index;
2570
2571   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2572   sm->hairpinning_node_index = node->index;
2573   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2574   sm->hairpin_dst_node_index = node->index;
2575   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2576   sm->hairpin_src_node_index = node->index;
2577   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2578   sm->ed_hairpinning_node_index = node->index;
2579   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2580   sm->ed_hairpin_dst_node_index = node->index;
2581   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2582   sm->ed_hairpin_src_node_index = node->index;
2583
2584   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2585   if (p)
2586     {
2587       tr = (vlib_thread_registration_t *) p[0];
2588       if (tr)
2589         {
2590           sm->num_workers = tr->count;
2591           sm->first_worker_index = tr->first_index;
2592         }
2593     }
2594
2595   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2596
2597   /* Use all available workers by default */
2598   if (sm->num_workers > 1)
2599     {
2600       for (i = 0; i < sm->num_workers; i++)
2601         bitmap = clib_bitmap_set (bitmap, i, 1);
2602       snat_set_workers (bitmap);
2603       clib_bitmap_free (bitmap);
2604     }
2605   else
2606     {
2607       sm->per_thread_data[0].snat_thread_index = 0;
2608     }
2609
2610   error = snat_api_init (vm, sm);
2611   if (error)
2612     return error;
2613
2614   /* Set up the interface address add/del callback */
2615   cb4.function = snat_ip4_add_del_interface_address_cb;
2616   cb4.function_opaque = 0;
2617
2618   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2619
2620   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2621   cb4.function_opaque = 0;
2622
2623   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2624
2625   nat_dpo_module_init ();
2626
2627   /* Init counters */
2628   sm->total_users.name = "total-users";
2629   sm->total_users.stat_segment_name = "/nat44/total-users";
2630   vlib_validate_simple_counter (&sm->total_users, 0);
2631   vlib_zero_simple_counter (&sm->total_users, 0);
2632   sm->total_sessions.name = "total-sessions";
2633   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2634   vlib_validate_simple_counter (&sm->total_sessions, 0);
2635   vlib_zero_simple_counter (&sm->total_sessions, 0);
2636
2637   /* Init IPFIX logging */
2638   snat_ipfix_logging_init (vm);
2639
2640   /* Init NAT64 */
2641   error = nat64_init (vm);
2642   if (error)
2643     return error;
2644
2645   nat66_init (vm);
2646
2647   ip4_table_bind_callback_t cbt4 = {
2648     .function = snat_ip4_table_bind,
2649   };
2650   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2651
2652   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2653                                         FIB_SOURCE_PRIORITY_HI,
2654                                         FIB_SOURCE_BH_SIMPLE);
2655   nat_fib_src_low = fib_source_allocate ("nat-low",
2656                                          FIB_SOURCE_PRIORITY_LOW,
2657                                          FIB_SOURCE_BH_SIMPLE);
2658
2659   test_ed_make_split ();
2660   return error;
2661 }
2662
2663 VLIB_INIT_FUNCTION (snat_init);
2664
2665 void
2666 snat_free_outside_address_and_port (snat_address_t * addresses,
2667                                     u32 thread_index, snat_session_key_t * k)
2668 {
2669   snat_address_t *a;
2670   u32 address_index;
2671   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2672
2673   for (address_index = 0; address_index < vec_len (addresses);
2674        address_index++)
2675     {
2676       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2677         break;
2678     }
2679
2680   ASSERT (address_index < vec_len (addresses));
2681
2682   a = addresses + address_index;
2683
2684   switch (k->protocol)
2685     {
2686 #define _(N, i, n, s) \
2687     case NAT_PROTOCOL_##N: \
2688       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2689       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2690       a->busy_##n##_ports--; \
2691       a->busy_##n##_ports_per_thread[thread_index]--; \
2692       break;
2693       foreach_nat_protocol
2694 #undef _
2695     default:
2696       nat_elog_info ("unknown protocol");
2697       return;
2698     }
2699 }
2700
2701 static int
2702 nat_set_outside_address_and_port (snat_address_t * addresses,
2703                                   u32 thread_index, snat_session_key_t * k)
2704 {
2705   snat_address_t *a = 0;
2706   u32 address_index;
2707   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2708
2709   for (address_index = 0; address_index < vec_len (addresses);
2710        address_index++)
2711     {
2712       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2713         continue;
2714
2715       a = addresses + address_index;
2716       switch (k->protocol)
2717         {
2718 #define _(N, j, n, s) \
2719         case NAT_PROTOCOL_##N: \
2720           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2721             return VNET_API_ERROR_INSTANCE_IN_USE; \
2722           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2723           a->busy_##n##_ports_per_thread[thread_index]++; \
2724           a->busy_##n##_ports++; \
2725           return 0;
2726           foreach_nat_protocol
2727 #undef _
2728         default:
2729           nat_elog_info ("unknown protocol");
2730           return 1;
2731         }
2732     }
2733
2734   return VNET_API_ERROR_NO_SUCH_ENTRY;
2735 }
2736
2737 int
2738 snat_static_mapping_match (snat_main_t * sm,
2739                            snat_session_key_t match,
2740                            snat_session_key_t * mapping,
2741                            u8 by_external,
2742                            u8 * is_addr_only,
2743                            twice_nat_type_t * twice_nat,
2744                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2745                            u8 * is_identity_nat)
2746 {
2747   clib_bihash_kv_8_8_t kv, value;
2748   snat_static_mapping_t *m;
2749   snat_session_key_t m_key;
2750   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2751   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2752   u8 backend_index;
2753   nat44_lb_addr_port_t *local;
2754
2755   m_key.fib_index = match.fib_index;
2756   if (by_external)
2757     {
2758       mapping_hash = &sm->static_mapping_by_external;
2759       m_key.fib_index = 0;
2760     }
2761
2762   m_key.addr = match.addr;
2763   m_key.port = clib_net_to_host_u16 (match.port);
2764   m_key.protocol = match.protocol;
2765
2766   kv.key = m_key.as_u64;
2767
2768   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2769     {
2770       /* Try address only mapping */
2771       m_key.port = 0;
2772       m_key.protocol = 0;
2773       kv.key = m_key.as_u64;
2774       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2775         return 1;
2776     }
2777
2778   m = pool_elt_at_index (sm->static_mappings, value.value);
2779
2780   if (by_external)
2781     {
2782       if (is_lb_static_mapping (m))
2783         {
2784           if (PREDICT_FALSE (lb != 0))
2785             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2786           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2787                                                           match.addr,
2788                                                           match.protocol,
2789                                                           match.port,
2790                                                           &backend_index))
2791             {
2792               local = pool_elt_at_index (m->locals, backend_index);
2793               mapping->addr = local->addr;
2794               mapping->port = clib_host_to_net_u16 (local->port);
2795               mapping->fib_index = local->fib_index;
2796               goto end;
2797             }
2798           // pick locals matching this worker
2799           if (PREDICT_FALSE (sm->num_workers > 1))
2800             {
2801               u32 thread_index = vlib_get_thread_index ();
2802               /* *INDENT-OFF* */
2803               pool_foreach_index (i, m->locals,
2804               ({
2805                 local = pool_elt_at_index (m->locals, i);
2806
2807                 ip4_header_t ip = {
2808                   .src_address = local->addr,
2809                 };
2810
2811                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2812                     thread_index)
2813                   {
2814                     vec_add1 (tmp, i);
2815                   }
2816               }));
2817               /* *INDENT-ON* */
2818               ASSERT (vec_len (tmp) != 0);
2819             }
2820           else
2821             {
2822               /* *INDENT-OFF* */
2823               pool_foreach_index (i, m->locals,
2824               ({
2825                 vec_add1 (tmp, i);
2826               }));
2827               /* *INDENT-ON* */
2828             }
2829           hi = vec_len (tmp) - 1;
2830           local = pool_elt_at_index (m->locals, tmp[hi]);
2831           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2832           while (lo < hi)
2833             {
2834               mid = ((hi - lo) >> 1) + lo;
2835               local = pool_elt_at_index (m->locals, tmp[mid]);
2836               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2837             }
2838           local = pool_elt_at_index (m->locals, tmp[lo]);
2839           if (!(local->prefix >= rand))
2840             return 1;
2841           mapping->addr = local->addr;
2842           mapping->port = clib_host_to_net_u16 (local->port);
2843           mapping->fib_index = local->fib_index;
2844           if (m->affinity)
2845             {
2846               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2847                                                 match.protocol, match.port,
2848                                                 tmp[lo], m->affinity,
2849                                                 m->affinity_per_service_list_head_index))
2850                 nat_elog_info ("create affinity record failed");
2851             }
2852           vec_free (tmp);
2853         }
2854       else
2855         {
2856           if (PREDICT_FALSE (lb != 0))
2857             *lb = NO_LB_NAT;
2858           mapping->fib_index = m->fib_index;
2859           mapping->addr = m->local_addr;
2860           /* Address only mapping doesn't change port */
2861           mapping->port = is_addr_only_static_mapping (m) ? match.port
2862             : clib_host_to_net_u16 (m->local_port);
2863         }
2864       mapping->protocol = m->proto;
2865     }
2866   else
2867     {
2868       mapping->addr = m->external_addr;
2869       /* Address only mapping doesn't change port */
2870       mapping->port = is_addr_only_static_mapping (m) ? match.port
2871         : clib_host_to_net_u16 (m->external_port);
2872       mapping->fib_index = sm->outside_fib_index;
2873     }
2874
2875 end:
2876   if (PREDICT_FALSE (is_addr_only != 0))
2877     *is_addr_only = is_addr_only_static_mapping (m);
2878
2879   if (PREDICT_FALSE (twice_nat != 0))
2880     *twice_nat = m->twice_nat;
2881
2882   if (PREDICT_FALSE (is_identity_nat != 0))
2883     *is_identity_nat = is_identity_static_mapping (m);
2884
2885   return 0;
2886 }
2887
2888 static_always_inline u16
2889 snat_random_port (u16 min, u16 max)
2890 {
2891   snat_main_t *sm = &snat_main;
2892   return min + random_u32 (&sm->random_seed) /
2893     (random_u32_max () / (max - min + 1) + 1);
2894 }
2895
2896 int
2897 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2898                                      u32 fib_index,
2899                                      u32 thread_index,
2900                                      snat_session_key_t * k,
2901                                      u16 port_per_thread,
2902                                      u32 snat_thread_index)
2903 {
2904   snat_main_t *sm = &snat_main;
2905
2906   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2907                                   port_per_thread, snat_thread_index);
2908 }
2909
2910 static int
2911 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2912                                  u32 fib_index,
2913                                  u32 thread_index,
2914                                  snat_session_key_t * k,
2915                                  u16 port_per_thread, u32 snat_thread_index)
2916 {
2917   int i;
2918   snat_address_t *a, *ga = 0;
2919   u32 portnum;
2920
2921   for (i = 0; i < vec_len (addresses); i++)
2922     {
2923       a = addresses + i;
2924       switch (k->protocol)
2925         {
2926 #define _(N, j, n, s) \
2927         case NAT_PROTOCOL_##N: \
2928           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2929             { \
2930               if (a->fib_index == fib_index) \
2931                 { \
2932                   while (1) \
2933                     { \
2934                       portnum = (port_per_thread * \
2935                         snat_thread_index) + \
2936                         snat_random_port(1, port_per_thread) + 1024; \
2937                       if (a->busy_##n##_port_refcounts[portnum]) \
2938                         continue; \
2939                       --a->busy_##n##_port_refcounts[portnum]; \
2940                       a->busy_##n##_ports_per_thread[thread_index]++; \
2941                       a->busy_##n##_ports++; \
2942                       k->addr = a->addr; \
2943                       k->port = clib_host_to_net_u16(portnum); \
2944                       return 0; \
2945                     } \
2946                 } \
2947               else if (a->fib_index == ~0) \
2948                 { \
2949                   ga = a; \
2950                 } \
2951             } \
2952           break;
2953           foreach_nat_protocol
2954 #undef _
2955         default:
2956           nat_elog_info ("unknown protocol");
2957           return 1;
2958         }
2959
2960     }
2961
2962   if (ga)
2963     {
2964       a = ga;
2965       switch (k->protocol)
2966         {
2967 #define _(N, j, n, s) \
2968         case NAT_PROTOCOL_##N: \
2969           while (1) \
2970             { \
2971               portnum = (port_per_thread * \
2972                 snat_thread_index) + \
2973                 snat_random_port(1, port_per_thread) + 1024; \
2974               if (a->busy_##n##_port_refcounts[portnum]) \
2975                 continue; \
2976               ++a->busy_##n##_port_refcounts[portnum]; \
2977               a->busy_##n##_ports_per_thread[thread_index]++; \
2978               a->busy_##n##_ports++; \
2979               k->addr = a->addr; \
2980               k->port = clib_host_to_net_u16(portnum); \
2981               return 0; \
2982             }
2983           break;
2984           foreach_nat_protocol
2985 #undef _
2986         default:
2987           nat_elog_info ("unknown protocol");
2988           return 1;
2989         }
2990     }
2991
2992   /* Totally out of translations to use... */
2993   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2994   return 1;
2995 }
2996
2997 static int
2998 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2999                               u32 fib_index,
3000                               u32 thread_index,
3001                               snat_session_key_t * k,
3002                               u16 port_per_thread, u32 snat_thread_index)
3003 {
3004   snat_main_t *sm = &snat_main;
3005   snat_address_t *a = addresses;
3006   u16 m, ports, portnum, A, j;
3007   m = 16 - (sm->psid_offset + sm->psid_length);
3008   ports = (1 << (16 - sm->psid_length)) - (1 << m);
3009
3010   if (!vec_len (addresses))
3011     goto exhausted;
3012
3013   switch (k->protocol)
3014     {
3015 #define _(N, i, n, s) \
3016     case NAT_PROTOCOL_##N: \
3017       if (a->busy_##n##_ports < ports) \
3018         { \
3019           while (1) \
3020             { \
3021               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3022               j = snat_random_port(0, pow2_mask(m)); \
3023               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3024               if (a->busy_##n##_port_refcounts[portnum]) \
3025                 continue; \
3026               ++a->busy_##n##_port_refcounts[portnum]; \
3027               a->busy_##n##_ports++; \
3028               k->addr = a->addr; \
3029               k->port = clib_host_to_net_u16 (portnum); \
3030               return 0; \
3031             } \
3032         } \
3033       break;
3034       foreach_nat_protocol
3035 #undef _
3036     default:
3037       nat_elog_info ("unknown protocol");
3038       return 1;
3039     }
3040
3041 exhausted:
3042   /* Totally out of translations to use... */
3043   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3044   return 1;
3045 }
3046
3047 static int
3048 nat_alloc_addr_and_port_range (snat_address_t * addresses,
3049                                u32 fib_index,
3050                                u32 thread_index,
3051                                snat_session_key_t * k,
3052                                u16 port_per_thread, u32 snat_thread_index)
3053 {
3054   snat_main_t *sm = &snat_main;
3055   snat_address_t *a = addresses;
3056   u16 portnum, ports;
3057
3058   ports = sm->end_port - sm->start_port + 1;
3059
3060   if (!vec_len (addresses))
3061     goto exhausted;
3062
3063   switch (k->protocol)
3064     {
3065 #define _(N, i, n, s) \
3066     case NAT_PROTOCOL_##N: \
3067       if (a->busy_##n##_ports < ports) \
3068         { \
3069           while (1) \
3070             { \
3071               portnum = snat_random_port(sm->start_port, sm->end_port); \
3072               if (a->busy_##n##_port_refcounts[portnum]) \
3073                 continue; \
3074               ++a->busy_##n##_port_refcounts[portnum]; \
3075               a->busy_##n##_ports++; \
3076               k->addr = a->addr; \
3077               k->port = clib_host_to_net_u16 (portnum); \
3078               return 0; \
3079             } \
3080         } \
3081       break;
3082       foreach_nat_protocol
3083 #undef _
3084     default:
3085       nat_elog_info ("unknown protocol");
3086       return 1;
3087     }
3088
3089 exhausted:
3090   /* Totally out of translations to use... */
3091   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3092   return 1;
3093 }
3094
3095 void
3096 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3097 {
3098   dpo_id_t dpo_v4 = DPO_INVALID;
3099   fib_prefix_t pfx = {
3100     .fp_proto = FIB_PROTOCOL_IP4,
3101     .fp_len = 32,
3102     .fp_addr.ip4.as_u32 = addr.as_u32,
3103   };
3104
3105   if (is_add)
3106     {
3107       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3108       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3109                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3110       dpo_reset (&dpo_v4);
3111     }
3112   else
3113     {
3114       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3115     }
3116 }
3117
3118 u8 *
3119 format_session_kvp (u8 * s, va_list * args)
3120 {
3121   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3122   snat_session_key_t k;
3123
3124   k.as_u64 = v->key;
3125
3126   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
3127
3128   return s;
3129 }
3130
3131 u8 *
3132 format_static_mapping_kvp (u8 * s, va_list * args)
3133 {
3134   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3135   snat_session_key_t k;
3136
3137   k.as_u64 = v->key;
3138
3139   s = format (s, "%U static-mapping-index %llu",
3140               format_static_mapping_key, &k, v->value);
3141
3142   return s;
3143 }
3144
3145 u8 *
3146 format_user_kvp (u8 * s, va_list * args)
3147 {
3148   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3149   snat_user_key_t k;
3150
3151   k.as_u64 = v->key;
3152
3153   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3154               k.fib_index, v->value);
3155
3156   return s;
3157 }
3158
3159 u8 *
3160 format_ed_session_kvp (u8 * s, va_list * args)
3161 {
3162   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3163
3164   u8 proto;
3165   u16 r_port, l_port;
3166   ip4_address_t l_addr, r_addr;
3167   u32 fib_index;
3168
3169   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
3170   s =
3171     format (s,
3172             "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
3173             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
3174             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
3175             format_ip_protocol, proto, fib_index,
3176             ed_value_get_session_index (v), ed_value_get_thread_index (v));
3177
3178   return s;
3179 }
3180
3181 static u32
3182 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3183                            u8 is_output)
3184 {
3185   snat_main_t *sm = &snat_main;
3186   u32 next_worker_index = 0;
3187   u32 hash;
3188
3189   next_worker_index = sm->first_worker_index;
3190   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3191     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3192
3193   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3194     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3195   else
3196     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3197
3198   return next_worker_index;
3199 }
3200
3201 static u32
3202 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3203                            u32 rx_fib_index0, u8 is_output)
3204 {
3205   snat_main_t *sm = &snat_main;
3206   udp_header_t *udp;
3207   u16 port;
3208   snat_session_key_t m_key;
3209   clib_bihash_kv_8_8_t kv, value;
3210   snat_static_mapping_t *m;
3211   u32 proto;
3212   u32 next_worker_index = 0;
3213
3214   /* first try static mappings without port */
3215   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3216     {
3217       m_key.addr = ip0->dst_address;
3218       m_key.port = 0;
3219       m_key.protocol = 0;
3220       m_key.fib_index = rx_fib_index0;
3221       kv.key = m_key.as_u64;
3222       if (!clib_bihash_search_8_8
3223           (&sm->static_mapping_by_external, &kv, &value))
3224         {
3225           m = pool_elt_at_index (sm->static_mappings, value.value);
3226           return m->workers[0];
3227         }
3228     }
3229
3230   proto = ip_proto_to_nat_proto (ip0->protocol);
3231   udp = ip4_next_header (ip0);
3232   port = udp->dst_port;
3233
3234   /* unknown protocol */
3235   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3236     {
3237       /* use current thread */
3238       return vlib_get_thread_index ();
3239     }
3240
3241   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3242     {
3243       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3244       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3245       if (!icmp_type_is_error_message
3246           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3247         port = vnet_buffer (b)->ip.reass.l4_src_port;
3248       else
3249         {
3250           /* if error message, then it's not fragmented and we can access it */
3251           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3252           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3253           void *l4_header = ip4_next_header (inner_ip);
3254           switch (proto)
3255             {
3256             case NAT_PROTOCOL_ICMP:
3257               icmp = (icmp46_header_t *) l4_header;
3258               echo = (icmp_echo_header_t *) (icmp + 1);
3259               port = echo->identifier;
3260               break;
3261             case NAT_PROTOCOL_UDP:
3262             case NAT_PROTOCOL_TCP:
3263               port = ((tcp_udp_header_t *) l4_header)->src_port;
3264               break;
3265             default:
3266               return vlib_get_thread_index ();
3267             }
3268         }
3269     }
3270
3271   /* try static mappings with port */
3272   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3273     {
3274       m_key.addr = ip0->dst_address;
3275       m_key.port = clib_net_to_host_u16 (port);
3276       m_key.protocol = proto;
3277       m_key.fib_index = rx_fib_index0;
3278       kv.key = m_key.as_u64;
3279       if (!clib_bihash_search_8_8
3280           (&sm->static_mapping_by_external, &kv, &value))
3281         {
3282           m = pool_elt_at_index (sm->static_mappings, value.value);
3283           return m->workers[0];
3284         }
3285     }
3286
3287   /* worker by outside port */
3288   next_worker_index = sm->first_worker_index;
3289   next_worker_index +=
3290     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3291   return next_worker_index;
3292 }
3293
3294 static u32
3295 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3296                                u8 is_output)
3297 {
3298   snat_main_t *sm = &snat_main;
3299   u32 next_worker_index = sm->first_worker_index;
3300   u32 hash;
3301
3302   clib_bihash_kv_16_8_t kv16, value16;
3303   snat_main_per_thread_data_t *tsm;
3304   udp_header_t *udp;
3305
3306   if (PREDICT_FALSE (is_output))
3307     {
3308       u32 fib_index = sm->outside_fib_index;
3309       nat_outside_fib_t *outside_fib;
3310       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3311       fib_prefix_t pfx = {
3312         .fp_proto = FIB_PROTOCOL_IP4,
3313         .fp_len = 32,
3314         .fp_addr = {
3315                     .ip4.as_u32 = ip->dst_address.as_u32,
3316                     }
3317         ,
3318       };
3319
3320       udp = ip4_next_header (ip);
3321
3322       switch (vec_len (sm->outside_fibs))
3323         {
3324         case 0:
3325           fib_index = sm->outside_fib_index;
3326           break;
3327         case 1:
3328           fib_index = sm->outside_fibs[0].fib_index;
3329           break;
3330         default:
3331             /* *INDENT-OFF* */
3332             vec_foreach (outside_fib, sm->outside_fibs)
3333               {
3334                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3335                 if (FIB_NODE_INDEX_INVALID != fei)
3336                   {
3337                     if (fib_entry_get_resolving_interface (fei) != ~0)
3338                       {
3339                         fib_index = outside_fib->fib_index;
3340                         break;
3341                       }
3342                   }
3343               }
3344             /* *INDENT-ON* */
3345           break;
3346         }
3347
3348       make_ed_kv (&ip->src_address, &ip->dst_address,
3349                   ip->protocol, fib_index, udp->src_port, udp->dst_port,
3350                   ~0, ~0, &kv16);
3351
3352       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3353                                                   &kv16, &value16)))
3354         {
3355           tsm =
3356             vec_elt_at_index (sm->per_thread_data,
3357                               ed_value_get_thread_index (&value16));
3358           next_worker_index += tsm->thread_index;
3359
3360           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3361                                   next_worker_index, fib_index,
3362                                   clib_net_to_host_u32 (ip->
3363                                                         src_address.as_u32),
3364                                   clib_net_to_host_u32 (ip->
3365                                                         dst_address.as_u32));
3366
3367           return next_worker_index;
3368         }
3369     }
3370
3371   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3372     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3373
3374   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3375     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3376   else
3377     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3378
3379   if (PREDICT_TRUE (!is_output))
3380     {
3381       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3382                               next_worker_index, rx_fib_index,
3383                               clib_net_to_host_u32 (ip->src_address.as_u32),
3384                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3385     }
3386   else
3387     {
3388       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3389                               next_worker_index, rx_fib_index,
3390                               clib_net_to_host_u32 (ip->src_address.as_u32),
3391                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3392     }
3393
3394   return next_worker_index;
3395 }
3396
3397 static u32
3398 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3399                                u32 rx_fib_index, u8 is_output)
3400 {
3401   snat_main_t *sm = &snat_main;
3402   clib_bihash_kv_8_8_t kv, value;
3403   clib_bihash_kv_16_8_t kv16, value16;
3404   snat_main_per_thread_data_t *tsm;
3405
3406   u32 proto, next_worker_index = 0;
3407   udp_header_t *udp;
3408   u16 port;
3409   snat_static_mapping_t *m;
3410   u32 hash;
3411
3412   proto = ip_proto_to_nat_proto (ip->protocol);
3413
3414   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3415     {
3416       udp = ip4_next_header (ip);
3417
3418       make_ed_kv (&ip->dst_address, &ip->src_address,
3419                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port,
3420                   ~0, ~0, &kv16);
3421
3422       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3423                                                   &kv16, &value16)))
3424         {
3425           tsm =
3426             vec_elt_at_index (sm->per_thread_data,
3427                               ed_value_get_thread_index (&value16));
3428           vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
3429             ed_value_get_session_index (&value16);
3430           next_worker_index = sm->first_worker_index + tsm->thread_index;
3431           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3432                                   next_worker_index, rx_fib_index,
3433                                   clib_net_to_host_u32 (ip->
3434                                                         src_address.as_u32),
3435                                   clib_net_to_host_u32 (ip->
3436                                                         dst_address.as_u32));
3437           return next_worker_index;
3438         }
3439     }
3440   else if (proto == NAT_PROTOCOL_ICMP)
3441     {
3442       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
3443         {
3444           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3445                                                       &kv16, &value16)))
3446             {
3447               tsm =
3448                 vec_elt_at_index (sm->per_thread_data,
3449                                   ed_value_get_thread_index (&value16));
3450               next_worker_index = sm->first_worker_index + tsm->thread_index;
3451               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3452                                       next_worker_index, rx_fib_index,
3453                                       clib_net_to_host_u32 (ip->
3454                                                             src_address.as_u32),
3455                                       clib_net_to_host_u32 (ip->
3456                                                             dst_address.as_u32));
3457               return next_worker_index;
3458             }
3459         }
3460     }
3461
3462   /* first try static mappings without port */
3463   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3464     {
3465       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3466       if (!clib_bihash_search_8_8
3467           (&sm->static_mapping_by_external, &kv, &value))
3468         {
3469           m = pool_elt_at_index (sm->static_mappings, value.value);
3470           next_worker_index = m->workers[0];
3471           goto done;
3472         }
3473     }
3474
3475   /* unknown protocol */
3476   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3477     {
3478       /* use current thread */
3479       next_worker_index = vlib_get_thread_index ();
3480       goto done;
3481     }
3482
3483   udp = ip4_next_header (ip);
3484   port = udp->dst_port;
3485
3486   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3487     {
3488       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3489       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3490       if (!icmp_type_is_error_message
3491           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3492         port = vnet_buffer (b)->ip.reass.l4_src_port;
3493       else
3494         {
3495           /* if error message, then it's not fragmented and we can access it */
3496           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3497           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3498           void *l4_header = ip4_next_header (inner_ip);
3499           switch (proto)
3500             {
3501             case NAT_PROTOCOL_ICMP:
3502               icmp = (icmp46_header_t *) l4_header;
3503               echo = (icmp_echo_header_t *) (icmp + 1);
3504               port = echo->identifier;
3505               break;
3506             case NAT_PROTOCOL_UDP:
3507             case NAT_PROTOCOL_TCP:
3508               port = ((tcp_udp_header_t *) l4_header)->src_port;
3509               break;
3510             default:
3511               next_worker_index = vlib_get_thread_index ();
3512               goto done;
3513             }
3514         }
3515     }
3516
3517   /* try static mappings with port */
3518   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3519     {
3520       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3521                   clib_net_to_host_u16 (port));
3522       if (!clib_bihash_search_8_8
3523           (&sm->static_mapping_by_external, &kv, &value))
3524         {
3525           m = pool_elt_at_index (sm->static_mappings, value.value);
3526           if (!is_lb_static_mapping (m))
3527             {
3528               next_worker_index = m->workers[0];
3529               goto done;
3530             }
3531
3532           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3533             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3534
3535           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3536             next_worker_index =
3537               m->workers[hash & (_vec_len (m->workers) - 1)];
3538           else
3539             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3540           goto done;
3541         }
3542     }
3543
3544   /* worker by outside port */
3545   next_worker_index = sm->first_worker_index;
3546   next_worker_index +=
3547     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3548
3549 done:
3550   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3551                           clib_net_to_host_u32 (ip->src_address.as_u32),
3552                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3553   return next_worker_index;
3554 }
3555
3556 void
3557 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3558                 ip4_address_t * out_addr, u16 out_port,
3559                 ip4_address_t * eh_addr, u16 eh_port,
3560                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3561                 u32 fib_index, u16 flags, u32 thread_index)
3562 {
3563   snat_main_t *sm = &snat_main;
3564   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3565   snat_session_key_t key;
3566   snat_user_t *u;
3567   snat_session_t *s;
3568   clib_bihash_kv_8_8_t kv;
3569   vlib_main_t *vm = vlib_get_main ();
3570   f64 now = vlib_time_now (vm);
3571   nat_outside_fib_t *outside_fib;
3572   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3573   fib_prefix_t pfx = {
3574     .fp_proto = FIB_PROTOCOL_IP4,
3575     .fp_len = 32,
3576     .fp_addr = {
3577                 .ip4.as_u32 = eh_addr->as_u32,
3578                 },
3579   };
3580
3581   key.addr.as_u32 = out_addr->as_u32;
3582   key.port = out_port;
3583   key.protocol = proto;
3584
3585   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3586     {
3587       if (nat_set_outside_address_and_port
3588           (sm->addresses, thread_index, &key))
3589         return;
3590     }
3591
3592   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3593   if (!u)
3594     return;
3595
3596   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3597   if (!s)
3598     return;
3599
3600   if (sm->endpoint_dependent)
3601     {
3602       nat_ed_lru_insert (tsm, s, now, proto);
3603     }
3604
3605   s->last_heard = now;
3606   s->flags = flags;
3607   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3608   s->ext_host_port = eh_port;
3609   user_session_increment (sm, u, snat_is_session_static (s));
3610   switch (vec_len (sm->outside_fibs))
3611     {
3612     case 0:
3613       key.fib_index = sm->outside_fib_index;
3614       break;
3615     case 1:
3616       key.fib_index = sm->outside_fibs[0].fib_index;
3617       break;
3618     default:
3619       /* *INDENT-OFF* */
3620       vec_foreach (outside_fib, sm->outside_fibs)
3621         {
3622           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3623           if (FIB_NODE_INDEX_INVALID != fei)
3624             {
3625               if (fib_entry_get_resolving_interface (fei) != ~0)
3626                 {
3627                   key.fib_index = outside_fib->fib_index;
3628                   break;
3629                 }
3630             }
3631         }
3632       /* *INDENT-ON* */
3633       break;
3634     }
3635   s->out2in = key;
3636   kv.key = key.as_u64;
3637   kv.value = s - tsm->sessions;
3638   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3639     nat_elog_warn ("out2in key add failed");
3640
3641   key.addr.as_u32 = in_addr->as_u32;
3642   key.port = in_port;
3643   key.fib_index = fib_index;
3644   s->in2out = key;
3645   kv.key = key.as_u64;
3646   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3647     nat_elog_warn ("in2out key add failed");
3648 }
3649
3650 void
3651 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3652                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3653                 u32 ti)
3654 {
3655   snat_main_t *sm = &snat_main;
3656   snat_session_key_t key;
3657   clib_bihash_kv_8_8_t kv, value;
3658   u32 thread_index;
3659   snat_session_t *s;
3660   snat_main_per_thread_data_t *tsm;
3661
3662   if (sm->num_workers > 1)
3663     thread_index =
3664       sm->first_worker_index +
3665       (sm->workers[(clib_net_to_host_u16 (out_port) -
3666                     1024) / sm->port_per_thread]);
3667   else
3668     thread_index = sm->num_workers;
3669   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3670
3671   key.addr.as_u32 = out_addr->as_u32;
3672   key.port = out_port;
3673   key.protocol = proto;
3674   key.fib_index = fib_index;
3675   kv.key = key.as_u64;
3676   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3677     return;
3678
3679   s = pool_elt_at_index (tsm->sessions, value.value);
3680   nat_free_session_data (sm, s, thread_index, 1);
3681   nat44_delete_session (sm, s, thread_index);
3682 }
3683
3684 void
3685 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3686                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3687                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3688 {
3689   snat_main_t *sm = &snat_main;
3690   snat_session_key_t key;
3691   clib_bihash_kv_8_8_t kv, value;
3692   snat_session_t *s;
3693   snat_main_per_thread_data_t *tsm;
3694
3695   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3696
3697   key.addr.as_u32 = out_addr->as_u32;
3698   key.port = out_port;
3699   key.protocol = proto;
3700   key.fib_index = fib_index;
3701   kv.key = key.as_u64;
3702   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3703     return;
3704
3705   s = pool_elt_at_index (tsm->sessions, value.value);
3706   s->total_pkts = total_pkts;
3707   s->total_bytes = total_bytes;
3708 }
3709
3710 void
3711 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3712                    ip4_address_t * out_addr, u16 out_port,
3713                    ip4_address_t * eh_addr, u16 eh_port,
3714                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3715                    u32 fib_index, u16 flags, u32 thread_index)
3716 {
3717   snat_main_t *sm = &snat_main;
3718   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3719   snat_session_key_t key;
3720   snat_session_t *s;
3721   clib_bihash_kv_16_8_t kv;
3722   vlib_main_t *vm = vlib_get_main ();
3723   f64 now = vlib_time_now (vm);
3724   nat_outside_fib_t *outside_fib;
3725   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3726   fib_prefix_t pfx = {
3727     .fp_proto = FIB_PROTOCOL_IP4,
3728     .fp_len = 32,
3729     .fp_addr = {
3730                 .ip4.as_u32 = eh_addr->as_u32,
3731                 },
3732   };
3733
3734   key.addr.as_u32 = out_addr->as_u32;
3735   key.port = out_port;
3736   key.protocol = proto;
3737
3738   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3739     {
3740       if (nat_set_outside_address_and_port
3741           (sm->addresses, thread_index, &key))
3742         return;
3743     }
3744
3745   key.addr.as_u32 = ehn_addr->as_u32;
3746   key.port = ehn_port;
3747   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3748     {
3749       if (nat_set_outside_address_and_port
3750           (sm->twice_nat_addresses, thread_index, &key))
3751         return;
3752     }
3753
3754   s = nat_ed_session_alloc (sm, thread_index, now, proto);
3755   if (!s)
3756     return;
3757
3758   s->last_heard = now;
3759   s->flags = flags;
3760   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3761   s->ext_host_nat_port = s->ext_host_port = eh_port;
3762   if (is_twice_nat_session (s))
3763     {
3764       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3765       s->ext_host_nat_port = ehn_port;
3766     }
3767   switch (vec_len (sm->outside_fibs))
3768     {
3769     case 0:
3770       key.fib_index = sm->outside_fib_index;
3771       break;
3772     case 1:
3773       key.fib_index = sm->outside_fibs[0].fib_index;
3774       break;
3775     default:
3776       /* *INDENT-OFF* */
3777       vec_foreach (outside_fib, sm->outside_fibs)
3778         {
3779           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3780           if (FIB_NODE_INDEX_INVALID != fei)
3781             {
3782               if (fib_entry_get_resolving_interface (fei) != ~0)
3783                 {
3784                   key.fib_index = outside_fib->fib_index;
3785                   break;
3786                 }
3787             }
3788         }
3789       /* *INDENT-ON* */
3790       break;
3791     }
3792   key.addr.as_u32 = out_addr->as_u32;
3793   key.port = out_port;
3794   s->out2in = key;
3795   kv.value = s - tsm->sessions;
3796
3797   key.addr.as_u32 = in_addr->as_u32;
3798   key.port = in_port;
3799   key.fib_index = fib_index;
3800   s->in2out = key;
3801
3802   make_ed_kv (in_addr, &s->ext_host_nat_addr,
3803               nat_proto_to_ip_proto (proto), fib_index, in_port,
3804               s->ext_host_nat_port, thread_index, s - tsm->sessions, &kv);
3805   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3806     nat_elog_warn ("in2out key add failed");
3807
3808   make_ed_kv (out_addr, eh_addr, nat_proto_to_ip_proto (proto),
3809               s->out2in.fib_index, out_port, eh_port, thread_index,
3810               s - tsm->sessions, &kv);
3811   if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 1))
3812     nat_elog_warn ("out2in key add failed");
3813 }
3814
3815 void
3816 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3817                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3818                    u32 fib_index, u32 ti)
3819 {
3820   snat_main_t *sm = &snat_main;
3821   clib_bihash_kv_16_8_t kv, value;
3822   u32 thread_index;
3823   snat_session_t *s;
3824   snat_main_per_thread_data_t *tsm;
3825
3826   if (sm->num_workers > 1)
3827     thread_index =
3828       sm->first_worker_index +
3829       (sm->workers[(clib_net_to_host_u16 (out_port) -
3830                     1024) / sm->port_per_thread]);
3831   else
3832     thread_index = sm->num_workers;
3833   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3834
3835   make_ed_kv (out_addr, eh_addr, proto, fib_index, out_port, eh_port, ~0, ~0,
3836               &kv);
3837   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3838     return;
3839
3840   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3841   nat_free_session_data (sm, s, thread_index, 1);
3842   nat44_delete_session (sm, s, thread_index);
3843 }
3844
3845 void
3846 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3847                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3848                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3849                    u32 thread_index)
3850 {
3851   snat_main_t *sm = &snat_main;
3852   clib_bihash_kv_16_8_t kv, value;
3853   snat_session_t *s;
3854   snat_main_per_thread_data_t *tsm;
3855
3856   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3857
3858   make_ed_kv (out_addr, eh_addr, proto, fib_index, out_port, eh_port, ~0, ~0,
3859               &kv);
3860   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3861     return;
3862
3863   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3864   s->total_pkts = total_pkts;
3865   s->total_bytes = total_bytes;
3866 }
3867
3868 void
3869 nat44_db_init (snat_main_per_thread_data_t * tsm)
3870 {
3871   snat_main_t *sm = &snat_main;
3872
3873   pool_alloc (tsm->sessions, sm->max_translations);
3874   pool_alloc (tsm->lru_pool, sm->max_translations);
3875
3876   dlist_elt_t *head;
3877
3878   pool_get (tsm->lru_pool, head);
3879   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3880   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3881
3882   pool_get (tsm->lru_pool, head);
3883   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3884   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3885
3886   pool_get (tsm->lru_pool, head);
3887   tsm->udp_lru_head_index = head - tsm->lru_pool;
3888   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3889
3890   pool_get (tsm->lru_pool, head);
3891   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3892   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3893
3894   pool_get (tsm->lru_pool, head);
3895   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3896   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3897
3898   if (sm->endpoint_dependent)
3899     {
3900       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3901                              sm->translation_buckets,
3902                              sm->translation_memory_size);
3903       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3904                                           format_ed_session_kvp);
3905     }
3906   else
3907     {
3908       clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3909                             sm->translation_buckets,
3910                             sm->translation_memory_size);
3911       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
3912       clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3913                             sm->translation_buckets,
3914                             sm->translation_memory_size);
3915       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
3916     }
3917
3918   // TODO: resolve static mappings (put only to !ED)
3919   pool_alloc (tsm->list_pool, sm->max_translations);
3920   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets,
3921                         sm->user_memory_size);
3922   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3923 }
3924
3925 void
3926 nat44_db_free (snat_main_per_thread_data_t * tsm)
3927 {
3928   snat_main_t *sm = &snat_main;
3929
3930   pool_free (tsm->sessions);
3931   pool_free (tsm->lru_pool);
3932
3933   if (sm->endpoint_dependent)
3934     {
3935       clib_bihash_free_16_8 (&tsm->in2out_ed);
3936     }
3937   else
3938     {
3939       clib_bihash_free_8_8 (&tsm->in2out);
3940       clib_bihash_free_8_8 (&tsm->out2in);
3941     }
3942
3943   // TODO: resolve static mappings (put only to !ED)
3944   pool_free (tsm->users);
3945   pool_free (tsm->list_pool);
3946   clib_bihash_free_8_8 (&tsm->user_hash);
3947 }
3948
3949 void
3950 nat44_sessions_clear ()
3951 {
3952   snat_main_t *sm = &snat_main;
3953   snat_main_per_thread_data_t *tsm;
3954
3955   if (sm->endpoint_dependent)
3956     {
3957       clib_bihash_free_16_8 (&sm->out2in_ed);
3958       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
3959                              clib_max (1, sm->num_workers) *
3960                              sm->translation_buckets,
3961                              clib_max (1, sm->num_workers) *
3962                              sm->translation_memory_size);
3963       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
3964                                           format_ed_session_kvp);
3965     }
3966
3967   /* *INDENT-OFF* */
3968   vec_foreach (tsm, sm->per_thread_data)
3969     {
3970       u32 ti;
3971
3972       nat44_db_free (tsm);
3973       nat44_db_init (tsm);
3974
3975       ti = tsm->snat_thread_index;
3976       vlib_set_simple_counter (&sm->total_users, ti, 0, 0);
3977       vlib_set_simple_counter (&sm->total_sessions, ti, 0, 0);
3978     }
3979   /* *INDENT-ON* */
3980 }
3981
3982 static clib_error_t *
3983 snat_config (vlib_main_t * vm, unformat_input_t * input)
3984 {
3985   snat_main_t *sm = &snat_main;
3986   nat66_main_t *nm = &nat66_main;
3987   snat_main_per_thread_data_t *tsm;
3988
3989   u32 static_mapping_buckets = 1024;
3990   uword static_mapping_memory_size = 64 << 20;
3991
3992   u32 nat64_bib_buckets = 1024;
3993   u32 nat64_bib_memory_size = 128 << 20;
3994
3995   u32 nat64_st_buckets = 2048;
3996   uword nat64_st_memory_size = 256 << 20;
3997
3998   u32 user_buckets = 128;
3999   uword user_memory_size = 64 << 20;
4000   u32 translation_buckets = 1024;
4001   uword translation_memory_size = 128 << 20;
4002
4003   u32 max_translations_per_user = ~0;
4004
4005   u32 outside_vrf_id = 0;
4006   u32 outside_ip6_vrf_id = 0;
4007   u32 inside_vrf_id = 0;
4008   u8 static_mapping_only = 0;
4009   u8 static_mapping_connection_tracking = 0;
4010
4011   u32 udp_timeout = SNAT_UDP_TIMEOUT;
4012   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
4013   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
4014   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
4015
4016   sm->deterministic = 0;
4017   sm->out2in_dpo = 0;
4018   sm->endpoint_dependent = 0;
4019
4020   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
4021     {
4022       if (unformat
4023           (input, "translation hash buckets %d", &translation_buckets))
4024         ;
4025       else if (unformat (input, "udp timeout %d", &udp_timeout))
4026         ;
4027       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
4028         ;
4029       else if (unformat (input, "tcp transitory timeout %d",
4030                          &tcp_transitory_timeout));
4031       else if (unformat (input, "tcp established timeout %d",
4032                          &tcp_established_timeout));
4033       else if (unformat (input, "translation hash memory %d",
4034                          &translation_memory_size));
4035       else if (unformat (input, "user hash buckets %d", &user_buckets))
4036         ;
4037       else if (unformat (input, "user hash memory %d", &user_memory_size))
4038         ;
4039       else if (unformat (input, "max translations per user %d",
4040                          &max_translations_per_user))
4041         ;
4042       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
4043         ;
4044       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
4045         ;
4046       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
4047         ;
4048       else if (unformat (input, "static mapping only"))
4049         {
4050           static_mapping_only = 1;
4051           if (unformat (input, "connection tracking"))
4052             static_mapping_connection_tracking = 1;
4053         }
4054       else if (unformat (input, "deterministic"))
4055         sm->deterministic = 1;
4056       else if (unformat (input, "nat64 bib hash buckets %d",
4057                          &nat64_bib_buckets))
4058         ;
4059       else if (unformat (input, "nat64 bib hash memory %d",
4060                          &nat64_bib_memory_size))
4061         ;
4062       else
4063         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
4064         ;
4065       else if (unformat (input, "nat64 st hash memory %d",
4066                          &nat64_st_memory_size))
4067         ;
4068       else if (unformat (input, "out2in dpo"))
4069         sm->out2in_dpo = 1;
4070       else if (unformat (input, "endpoint-dependent"))
4071         sm->endpoint_dependent = 1;
4072       else
4073         return clib_error_return (0, "unknown input '%U'",
4074                                   format_unformat_error, input);
4075     }
4076
4077   if (sm->deterministic && sm->endpoint_dependent)
4078     return clib_error_return (0,
4079                               "deterministic and endpoint-dependent modes are mutually exclusive");
4080
4081   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
4082     return clib_error_return (0,
4083                               "static mapping only mode available only for simple nat");
4084
4085   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
4086     return clib_error_return (0,
4087                               "out2in dpo mode available only for simple nat");
4088
4089   /* optionally configurable timeouts for testing purposes */
4090   sm->udp_timeout = udp_timeout;
4091   sm->tcp_transitory_timeout = tcp_transitory_timeout;
4092   sm->tcp_established_timeout = tcp_established_timeout;
4093   sm->icmp_timeout = icmp_timeout;
4094
4095   sm->user_buckets = user_buckets;
4096   sm->user_memory_size = user_memory_size;
4097
4098   sm->translation_buckets = translation_buckets;
4099   sm->translation_memory_size = translation_memory_size;
4100   /* do not exceed load factor 10 */
4101   sm->max_translations = 10 * translation_buckets;
4102   vec_add1 (sm->max_translations_per_fib, sm->max_translations);
4103
4104   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4105     sm->max_translations : max_translations_per_user;
4106
4107   sm->outside_vrf_id = outside_vrf_id;
4108   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4109                                                              outside_vrf_id,
4110                                                              nat_fib_src_hi);
4111   nm->outside_vrf_id = outside_ip6_vrf_id;
4112   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
4113                                                              outside_ip6_vrf_id,
4114                                                              nat_fib_src_hi);
4115   sm->inside_vrf_id = inside_vrf_id;
4116   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4117                                                             inside_vrf_id,
4118                                                             nat_fib_src_hi);
4119   sm->static_mapping_only = static_mapping_only;
4120   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4121
4122   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4123                   nat64_st_memory_size);
4124
4125   if (sm->deterministic)
4126     {
4127       sm->in2out_node_index = snat_det_in2out_node.index;
4128       sm->in2out_output_node_index = ~0;
4129       sm->out2in_node_index = snat_det_out2in_node.index;
4130       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4131       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4132     }
4133   else
4134     {
4135       if (sm->endpoint_dependent)
4136         {
4137           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4138           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4139
4140           sm->in2out_node_index = nat44_ed_in2out_node.index;
4141           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4142           sm->out2in_node_index = nat44_ed_out2in_node.index;
4143
4144           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4145           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4146           nat_affinity_init (vm);
4147           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4148                        nat_ha_sref_ed_cb);
4149           clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4150                                  translation_buckets,
4151                                  translation_memory_size);
4152           clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4153                                               format_ed_session_kvp);
4154         }
4155       else
4156         {
4157           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4158           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4159
4160           sm->in2out_node_index = snat_in2out_node.index;
4161           sm->in2out_output_node_index = snat_in2out_output_node.index;
4162           sm->out2in_node_index = snat_out2in_node.index;
4163
4164           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4165           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4166           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4167         }
4168       if (!static_mapping_only ||
4169           (static_mapping_only && static_mapping_connection_tracking))
4170         {
4171           /* *INDENT-OFF* */
4172           vec_foreach (tsm, sm->per_thread_data)
4173             {
4174               nat44_db_init (tsm);
4175             }
4176           /* *INDENT-ON* */
4177         }
4178       else
4179         {
4180           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4181           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4182         }
4183       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4184                             "static_mapping_by_local", static_mapping_buckets,
4185                             static_mapping_memory_size);
4186       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4187                                          format_static_mapping_kvp);
4188
4189       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4190                             "static_mapping_by_external",
4191                             static_mapping_buckets,
4192                             static_mapping_memory_size);
4193       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4194                                          format_static_mapping_kvp);
4195     }
4196
4197   return 0;
4198 }
4199
4200 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4201
4202 static void
4203 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4204                                  uword opaque,
4205                                  u32 sw_if_index,
4206                                  ip4_address_t * address,
4207                                  u32 address_length,
4208                                  u32 if_address_index, u32 is_delete)
4209 {
4210   snat_main_t *sm = &snat_main;
4211   snat_static_map_resolve_t *rp;
4212   snat_static_mapping_t *m;
4213   snat_session_key_t m_key;
4214   clib_bihash_kv_8_8_t kv, value;
4215   int i, rv;
4216   ip4_address_t l_addr;
4217
4218   for (i = 0; i < vec_len (sm->to_resolve); i++)
4219     {
4220       rp = sm->to_resolve + i;
4221       if (rp->addr_only == 0)
4222         continue;
4223       if (rp->sw_if_index == sw_if_index)
4224         goto match;
4225     }
4226
4227   return;
4228
4229 match:
4230   m_key.addr.as_u32 = address->as_u32;
4231   m_key.port = rp->addr_only ? 0 : rp->e_port;
4232   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4233   m_key.fib_index = sm->outside_fib_index;
4234   kv.key = m_key.as_u64;
4235   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4236     m = 0;
4237   else
4238     m = pool_elt_at_index (sm->static_mappings, value.value);
4239
4240   if (!is_delete)
4241     {
4242       /* Don't trip over lease renewal, static config */
4243       if (m)
4244         return;
4245     }
4246   else
4247     {
4248       if (!m)
4249         return;
4250     }
4251
4252   /* Indetity mapping? */
4253   if (rp->l_addr.as_u32 == 0)
4254     l_addr.as_u32 = address[0].as_u32;
4255   else
4256     l_addr.as_u32 = rp->l_addr.as_u32;
4257   /* Add the static mapping */
4258   rv = snat_add_static_mapping (l_addr,
4259                                 address[0],
4260                                 rp->l_port,
4261                                 rp->e_port,
4262                                 rp->vrf_id,
4263                                 rp->addr_only, ~0 /* sw_if_index */ ,
4264                                 rp->proto, !is_delete, rp->twice_nat,
4265                                 rp->out2in_only, rp->tag, rp->identity_nat);
4266   if (rv)
4267     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4268 }
4269
4270 static void
4271 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4272                                        uword opaque,
4273                                        u32 sw_if_index,
4274                                        ip4_address_t * address,
4275                                        u32 address_length,
4276                                        u32 if_address_index, u32 is_delete)
4277 {
4278   snat_main_t *sm = &snat_main;
4279   snat_static_map_resolve_t *rp;
4280   ip4_address_t l_addr;
4281   int i, j;
4282   int rv;
4283   u8 twice_nat = 0;
4284   snat_address_t *addresses = sm->addresses;
4285
4286   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4287     {
4288       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4289         goto match;
4290     }
4291
4292   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4293     {
4294       twice_nat = 1;
4295       addresses = sm->twice_nat_addresses;
4296       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4297         goto match;
4298     }
4299
4300   return;
4301
4302 match:
4303   if (!is_delete)
4304     {
4305       /* Don't trip over lease renewal, static config */
4306       for (j = 0; j < vec_len (addresses); j++)
4307         if (addresses[j].addr.as_u32 == address->as_u32)
4308           return;
4309
4310       (void) snat_add_address (sm, address, ~0, twice_nat);
4311       /* Scan static map resolution vector */
4312       for (j = 0; j < vec_len (sm->to_resolve); j++)
4313         {
4314           rp = sm->to_resolve + j;
4315           if (rp->addr_only)
4316             continue;
4317           /* On this interface? */
4318           if (rp->sw_if_index == sw_if_index)
4319             {
4320               /* Indetity mapping? */
4321               if (rp->l_addr.as_u32 == 0)
4322                 l_addr.as_u32 = address[0].as_u32;
4323               else
4324                 l_addr.as_u32 = rp->l_addr.as_u32;
4325               /* Add the static mapping */
4326               rv = snat_add_static_mapping (l_addr,
4327                                             address[0],
4328                                             rp->l_port,
4329                                             rp->e_port,
4330                                             rp->vrf_id,
4331                                             rp->addr_only,
4332                                             ~0 /* sw_if_index */ ,
4333                                             rp->proto,
4334                                             rp->is_add, rp->twice_nat,
4335                                             rp->out2in_only, rp->tag,
4336                                             rp->identity_nat);
4337               if (rv)
4338                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4339                                     "i4", rv);
4340             }
4341         }
4342       return;
4343     }
4344   else
4345     {
4346       (void) snat_del_address (sm, address[0], 1, twice_nat);
4347       return;
4348     }
4349 }
4350
4351
4352 int
4353 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4354                             u8 twice_nat)
4355 {
4356   ip4_main_t *ip4_main = sm->ip4_main;
4357   ip4_address_t *first_int_addr;
4358   snat_static_map_resolve_t *rp;
4359   u32 *indices_to_delete = 0;
4360   int i, j;
4361   u32 *auto_add_sw_if_indices =
4362     twice_nat ? sm->
4363     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4364
4365   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4366     );
4367
4368   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4369     {
4370       if (auto_add_sw_if_indices[i] == sw_if_index)
4371         {
4372           if (is_del)
4373             {
4374               /* if have address remove it */
4375               if (first_int_addr)
4376                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4377               else
4378                 {
4379                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4380                     {
4381                       rp = sm->to_resolve + j;
4382                       if (rp->sw_if_index == sw_if_index)
4383                         vec_add1 (indices_to_delete, j);
4384                     }
4385                   if (vec_len (indices_to_delete))
4386                     {
4387                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4388                         vec_del1 (sm->to_resolve, j);
4389                       vec_free (indices_to_delete);
4390                     }
4391                 }
4392               if (twice_nat)
4393                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4394               else
4395                 vec_del1 (sm->auto_add_sw_if_indices, i);
4396             }
4397           else
4398             return VNET_API_ERROR_VALUE_EXIST;
4399
4400           return 0;
4401         }
4402     }
4403
4404   if (is_del)
4405     return VNET_API_ERROR_NO_SUCH_ENTRY;
4406
4407   /* add to the auto-address list */
4408   if (twice_nat)
4409     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4410   else
4411     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4412
4413   /* If the address is already bound - or static - add it now */
4414   if (first_int_addr)
4415     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4416
4417   return 0;
4418 }
4419
4420 int
4421 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4422                    nat_protocol_t proto, u32 vrf_id, int is_in)
4423 {
4424   snat_main_per_thread_data_t *tsm;
4425   clib_bihash_kv_8_8_t kv, value;
4426   ip4_header_t ip;
4427   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4428   snat_session_key_t key;
4429   snat_session_t *s;
4430   clib_bihash_8_8_t *t;
4431
4432   if (sm->endpoint_dependent)
4433     return VNET_API_ERROR_UNSUPPORTED;
4434
4435   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4436   if (sm->num_workers > 1)
4437     tsm =
4438       vec_elt_at_index (sm->per_thread_data,
4439                         sm->worker_in2out_cb (&ip, fib_index, 0));
4440   else
4441     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4442
4443   key.addr.as_u32 = addr->as_u32;
4444   key.port = clib_host_to_net_u16 (port);
4445   key.protocol = proto;
4446   key.fib_index = fib_index;
4447   kv.key = key.as_u64;
4448   t = is_in ? &tsm->in2out : &tsm->out2in;
4449   if (!clib_bihash_search_8_8 (t, &kv, &value))
4450     {
4451       if (pool_is_free_index (tsm->sessions, value.value))
4452         return VNET_API_ERROR_UNSPECIFIED;
4453
4454       s = pool_elt_at_index (tsm->sessions, value.value);
4455       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4456       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4457       return 0;
4458     }
4459
4460   return VNET_API_ERROR_NO_SUCH_ENTRY;
4461 }
4462
4463 int
4464 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4465                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4466                       u32 vrf_id, int is_in)
4467 {
4468   ip4_header_t ip;
4469   clib_bihash_16_8_t *t;
4470   clib_bihash_kv_16_8_t kv, value;
4471   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4472   snat_session_t *s;
4473   snat_main_per_thread_data_t *tsm;
4474
4475   if (!sm->endpoint_dependent)
4476     return VNET_API_ERROR_FEATURE_DISABLED;
4477
4478   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4479   if (sm->num_workers > 1)
4480     tsm =
4481       vec_elt_at_index (sm->per_thread_data,
4482                         sm->worker_in2out_cb (&ip, fib_index, 0));
4483   else
4484     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4485
4486   t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
4487   make_ed_kv (addr, eh_addr, proto, fib_index, clib_host_to_net_u16 (port),
4488               clib_host_to_net_u16 (eh_port), ~0, ~0, &kv);
4489   if (clib_bihash_search_16_8 (t, &kv, &value))
4490     {
4491       return VNET_API_ERROR_NO_SUCH_ENTRY;
4492     }
4493
4494   if (pool_is_free_index (tsm->sessions, value.value))
4495     return VNET_API_ERROR_UNSPECIFIED;
4496   s = pool_elt_at_index (tsm->sessions, value.value);
4497   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4498   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
4499   return 0;
4500 }
4501
4502 void
4503 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4504 {
4505   snat_main_t *sm = &snat_main;
4506
4507   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4508   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4509   sm->psid = psid;
4510   sm->psid_offset = psid_offset;
4511   sm->psid_length = psid_length;
4512 }
4513
4514 void
4515 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4516 {
4517   snat_main_t *sm = &snat_main;
4518
4519   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4520   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4521   sm->start_port = start_port;
4522   sm->end_port = end_port;
4523 }
4524
4525 void
4526 nat_set_alloc_addr_and_port_default (void)
4527 {
4528   snat_main_t *sm = &snat_main;
4529
4530   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4531   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4532 }
4533
4534 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4535                                  vlib_node_runtime_t * node,
4536                                  vlib_frame_t * frame)
4537 {
4538   return 0;
4539 }
4540
4541 /* *INDENT-OFF* */
4542 VLIB_REGISTER_NODE (nat_default_node) = {
4543   .name = "nat-default",
4544   .vector_size = sizeof (u32),
4545   .format_trace = 0,
4546   .type = VLIB_NODE_TYPE_INTERNAL,
4547   .n_errors = 0,
4548   .n_next_nodes = NAT_N_NEXT,
4549   .next_nodes = {
4550     [NAT_NEXT_DROP] = "error-drop",
4551     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4552     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4553     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4554     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4555     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4556     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4557     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4558     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4559   },
4560 };
4561 /* *INDENT-ON* */
4562
4563 /*
4564  * fd.io coding-style-patch-verification: ON
4565  *
4566  * Local Variables:
4567  * eval: (c-set-style "gnu")
4568  * End:
4569  */