src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 fib_source_t nat_fib_src_hi;
  42 fib_source_t nat_fib_src_low;
  43
  44 /* *INDENT-OFF* */
  45 /* Hook up input features */
  46 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat-pre-in2out",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-sv-reassembly-feature"),
  51 };
  52 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat-pre-out2in",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  56                                "ip4-dhcp-client-detect",
  57                                "ip4-sv-reassembly-feature"),
  58 };
  59 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-in2out-worker-handoff",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-out2in-worker-handoff",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-in2out",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  74 };
  75 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-out2in",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  79                                "ip4-dhcp-client-detect"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-classify",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  85 };
  86 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-det-in2out",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-det-out2in",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  95                                "ip4-dhcp-client-detect"),
  96 };
  97 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-det-classify",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-ed-in2out",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 106 };
 107 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-ed-out2in",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 111                                "ip4-dhcp-client-detect"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-ed-classify",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-handoff-classify",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 122 };
 123 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-in2out-fast",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 127 };
 128 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 129   .arc_name = "ip4-unicast",
 130   .node_name = "nat44-out2in-fast",
 131   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 132                                "ip4-dhcp-client-detect"),
 133 };
 134 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 135   .arc_name = "ip4-unicast",
 136   .node_name = "nat44-hairpin-dst",
 137   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 140   .arc_name = "ip4-unicast",
 141   .node_name = "nat44-ed-hairpin-dst",
 142   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 143 };
 144
 145 /* Hook up output features */
 146 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-in2out-output",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-in2out-output-worker-handoff",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 155 };
 156 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 157   .arc_name = "ip4-output",
 158   .node_name = "nat44-hairpin-src",
 159   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 162   .arc_name = "ip4-output",
 163   .node_name = "nat44-ed-in2out-output",
 164   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 165   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 166 };
 167 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 168   .arc_name = "ip4-output",
 169   .node_name = "nat44-ed-hairpin-src",
 170   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 171   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 172 };
 173
 174 /* Hook up ip4-local features */
 175 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 176 {
 177   .arc_name = "ip4-local",
 178   .node_name = "nat44-hairpinning",
 179   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 180 };
 181 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 182 {
 183   .arc_name = "ip4-local",
 184   .node_name = "nat44-ed-hairpinning",
 185   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 186 };
 187
 188
 189 VLIB_PLUGIN_REGISTER () = {
 190     .version = VPP_BUILD_VER,
 191     .description = "Network Address Translation (NAT)",
 192 };
 193 /* *INDENT-ON* */
 194
 195 void
 196 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 197                        u8 is_ha)
 198 {
 199   snat_session_key_t key;
 200   clib_bihash_kv_8_8_t kv;
 201   nat_ed_ses_key_t ed_key;
 202   clib_bihash_kv_16_8_t ed_kv;
 203   snat_main_per_thread_data_t *tsm =
 204     vec_elt_at_index (sm->per_thread_data, thread_index);
 205
 206   if (is_fwd_bypass_session (s))
 207     {
 208       if (snat_is_unk_proto_session (s))
 209         {
 210           ed_key.proto = s->in2out.port;
 211           ed_key.r_port = 0;
 212           ed_key.l_port = 0;
 213         }
 214       else
 215         {
 216           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 217           ed_key.l_port = s->in2out.port;
 218           ed_key.r_port = s->ext_host_port;
 219         }
 220       ed_key.l_addr = s->in2out.addr;
 221       ed_key.r_addr = s->ext_host_addr;
 222       ed_key.fib_index = 0;
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 226         nat_elog_warn ("in2out_ed key del failed");
 227       return;
 228     }
 229
 230   /* session lookup tables */
 231   if (is_ed_session (s))
 232     {
 233       if (is_affinity_sessions (s))
 234         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 235                              s->in2out.protocol, s->out2in.port);
 236       ed_key.l_addr = s->out2in.addr;
 237       ed_key.r_addr = s->ext_host_addr;
 238       ed_key.fib_index = s->out2in.fib_index;
 239       if (snat_is_unk_proto_session (s))
 240         {
 241           ed_key.proto = s->in2out.port;
 242           ed_key.r_port = 0;
 243           ed_key.l_port = 0;
 244         }
 245       else
 246         {
 247           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 248           ed_key.l_port = s->out2in.port;
 249           ed_key.r_port = s->ext_host_port;
 250         }
 251       ed_kv.key[0] = ed_key.as_u64[0];
 252       ed_kv.key[1] = ed_key.as_u64[1];
 253       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 254         nat_elog_warn ("out2in_ed key del failed");
 255       ed_key.l_addr = s->in2out.addr;
 256       ed_key.fib_index = s->in2out.fib_index;
 257       if (!snat_is_unk_proto_session (s))
 258         ed_key.l_port = s->in2out.port;
 259       if (is_twice_nat_session (s))
 260         {
 261           ed_key.r_addr = s->ext_host_nat_addr;
 262           ed_key.r_port = s->ext_host_nat_port;
 263         }
 264       ed_kv.key[0] = ed_key.as_u64[0];
 265       ed_kv.key[1] = ed_key.as_u64[1];
 266       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 267         nat_elog_warn ("in2out_ed key del failed");
 268
 269       if (!is_ha)
 270         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 271                                &s->in2out.addr, s->in2out.port,
 272                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 273                                &s->out2in.addr, s->out2in.port,
 274                                &s->ext_host_addr, s->ext_host_port,
 275                                s->in2out.protocol, is_twice_nat_session (s));
 276     }
 277   else
 278     {
 279       kv.key = s->in2out.as_u64;
 280       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 281         nat_elog_warn ("in2out key del failed");
 282       kv.key = s->out2in.as_u64;
 283       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 284         nat_elog_warn ("out2in key del failed");
 285
 286       if (!is_ha)
 287         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 288                                  &s->in2out.addr, s->in2out.port,
 289                                  &s->out2in.addr, s->out2in.port,
 290                                  s->in2out.protocol);
 291     }
 292
 293   if (snat_is_unk_proto_session (s))
 294     return;
 295
 296   if (!is_ha)
 297     {
 298       /* log NAT event */
 299       snat_ipfix_logging_nat44_ses_delete (thread_index,
 300                                            s->in2out.addr.as_u32,
 301                                            s->out2in.addr.as_u32,
 302                                            s->in2out.protocol,
 303                                            s->in2out.port,
 304                                            s->out2in.port,
 305                                            s->in2out.fib_index);
 306
 307       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 308                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 309                    thread_index);
 310     }
 311
 312   /* Twice NAT address and port for external host */
 313   if (is_twice_nat_session (s))
 314     {
 315       key.protocol = s->in2out.protocol;
 316       key.port = s->ext_host_nat_port;
 317       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 318       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 319                                           thread_index, &key);
 320     }
 321
 322   if (snat_is_session_static (s))
 323     return;
 324
 325   snat_free_outside_address_and_port (sm->addresses, thread_index,
 326                                       &s->out2in);
 327 }
 328
 329 void
 330 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 331                          u32 thread_index, u8 is_ha)
 332 {
 333   snat_session_key_t key;
 334   nat_ed_ses_key_t ed_key;
 335   clib_bihash_kv_16_8_t ed_kv;
 336   snat_main_per_thread_data_t *tsm =
 337     vec_elt_at_index (sm->per_thread_data, thread_index);
 338
 339   if (is_fwd_bypass_session (s))
 340     {
 341       if (snat_is_unk_proto_session (s))
 342         {
 343           ed_key.proto = s->in2out.port;
 344           ed_key.r_port = 0;
 345           ed_key.l_port = 0;
 346         }
 347       else
 348         {
 349           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 350           ed_key.l_port = s->in2out.port;
 351           ed_key.r_port = s->ext_host_port;
 352         }
 353
 354       ed_key.l_addr = s->in2out.addr;
 355       ed_key.r_addr = s->ext_host_addr;
 356       ed_key.fib_index = 0;
 357       ed_kv.key[0] = ed_key.as_u64[0];
 358       ed_kv.key[1] = ed_key.as_u64[1];
 359
 360       if (PREDICT_FALSE
 361           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 362         nat_elog_warn ("in2out_ed key del failed");
 363       return;
 364     }
 365
 366   /* session lookup tables */
 367   if (is_affinity_sessions (s))
 368     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 369                          s->in2out.protocol, s->out2in.port);
 370   ed_key.l_addr = s->out2in.addr;
 371   ed_key.r_addr = s->ext_host_addr;
 372   ed_key.fib_index = s->out2in.fib_index;
 373   if (snat_is_unk_proto_session (s))
 374     {
 375       ed_key.proto = s->in2out.port;
 376       ed_key.r_port = 0;
 377       ed_key.l_port = 0;
 378     }
 379   else
 380     {
 381       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 382       ed_key.l_port = s->out2in.port;
 383       ed_key.r_port = s->ext_host_port;
 384     }
 385   ed_kv.key[0] = ed_key.as_u64[0];
 386   ed_kv.key[1] = ed_key.as_u64[1];
 387
 388   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0)))
 389     nat_elog_warn ("out2in_ed key del failed");
 390
 391   ed_key.l_addr = s->in2out.addr;
 392   ed_key.fib_index = s->in2out.fib_index;
 393
 394   if (!snat_is_unk_proto_session (s))
 395     ed_key.l_port = s->in2out.port;
 396
 397   if (is_twice_nat_session (s))
 398     {
 399       ed_key.r_addr = s->ext_host_nat_addr;
 400       ed_key.r_port = s->ext_host_nat_port;
 401     }
 402
 403   ed_kv.key[0] = ed_key.as_u64[0];
 404   ed_kv.key[1] = ed_key.as_u64[1];
 405
 406   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 407     nat_elog_warn ("in2out_ed key del failed");
 408
 409   if (!is_ha)
 410     {
 411       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 412                              &s->in2out.addr, s->in2out.port,
 413                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 414                              &s->out2in.addr, s->out2in.port,
 415                              &s->ext_host_addr, s->ext_host_port,
 416                              s->in2out.protocol, is_twice_nat_session (s));
 417     }
 418
 419   if (snat_is_unk_proto_session (s))
 420     return;
 421
 422   // is this correct ?
 423   if (!is_ha)
 424     {
 425       snat_ipfix_logging_nat44_ses_delete (thread_index,
 426                                            s->in2out.addr.as_u32,
 427                                            s->out2in.addr.as_u32,
 428                                            s->in2out.protocol,
 429                                            s->in2out.port,
 430                                            s->out2in.port,
 431                                            s->in2out.fib_index);
 432       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 433                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 434                    thread_index);
 435     }
 436
 437   /* Twice NAT address and port for external host */
 438   if (is_twice_nat_session (s))
 439     {
 440       key.protocol = s->in2out.protocol;
 441       key.port = s->ext_host_nat_port;
 442       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 443       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 444                                           thread_index, &key);
 445     }
 446
 447   if (snat_is_session_static (s))
 448     return;
 449
 450   // should be called for every dynamic session
 451   snat_free_outside_address_and_port (sm->addresses, thread_index,
 452                                       &s->out2in);
 453 }
 454
 455
 456 snat_user_t *
 457 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 458                         u32 thread_index)
 459 {
 460   snat_user_t *u = 0;
 461   snat_user_key_t user_key;
 462   clib_bihash_kv_8_8_t kv, value;
 463   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 464   dlist_elt_t *per_user_list_head_elt;
 465
 466   user_key.addr.as_u32 = addr->as_u32;
 467   user_key.fib_index = fib_index;
 468   kv.key = user_key.as_u64;
 469
 470   /* Ever heard of the "user" = src ip4 address before? */
 471   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 472     {
 473       /* no, make a new one */
 474       pool_get (tsm->users, u);
 475       clib_memset (u, 0, sizeof (*u));
 476
 477       u->min_session_timeout = 0;
 478
 479       u->addr.as_u32 = addr->as_u32;
 480       u->fib_index = fib_index;
 481
 482       pool_get (tsm->list_pool, per_user_list_head_elt);
 483
 484       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 485         tsm->list_pool;
 486
 487       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 488
 489       kv.value = u - tsm->users;
 490
 491       /* add user */
 492       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 493         {
 494           nat_elog_warn ("user_hash key add failed");
 495           nat44_delete_user_with_no_session (sm, u, thread_index);
 496           return NULL;
 497         }
 498
 499       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 500                                pool_elts (tsm->users));
 501     }
 502   else
 503     {
 504       u = pool_elt_at_index (tsm->users, value.value);
 505     }
 506
 507   return u;
 508 }
 509
 510 snat_session_t *
 511 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 512                               u32 thread_index, f64 now)
 513 {
 514   snat_session_t *s;
 515   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 516   u32 oldest_per_user_translation_list_index, session_index;
 517   dlist_elt_t *oldest_per_user_translation_list_elt;
 518   dlist_elt_t *per_user_translation_list_elt;
 519
 520   /* Over quota? Recycle the least recently used translation */
 521   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 522     {
 523       oldest_per_user_translation_list_index =
 524         clib_dlist_remove_head (tsm->list_pool,
 525                                 u->sessions_per_user_list_head_index);
 526
 527       ASSERT (oldest_per_user_translation_list_index != ~0);
 528
 529       /* Add it back to the end of the LRU list */
 530       clib_dlist_addtail (tsm->list_pool,
 531                           u->sessions_per_user_list_head_index,
 532                           oldest_per_user_translation_list_index);
 533       /* Get the list element */
 534       oldest_per_user_translation_list_elt =
 535         pool_elt_at_index (tsm->list_pool,
 536                            oldest_per_user_translation_list_index);
 537
 538       /* Get the session index from the list element */
 539       session_index = oldest_per_user_translation_list_elt->value;
 540
 541       /* Get the session */
 542       s = pool_elt_at_index (tsm->sessions, session_index);
 543       nat_free_session_data (sm, s, thread_index, 0);
 544       if (snat_is_session_static (s))
 545         u->nstaticsessions--;
 546       else
 547         u->nsessions--;
 548       s->flags = 0;
 549       s->total_bytes = 0;
 550       s->total_pkts = 0;
 551       s->state = 0;
 552       s->ext_host_addr.as_u32 = 0;
 553       s->ext_host_port = 0;
 554       s->ext_host_nat_addr.as_u32 = 0;
 555       s->ext_host_nat_port = 0;
 556     }
 557   else
 558     {
 559       pool_get (tsm->sessions, s);
 560       clib_memset (s, 0, sizeof (*s));
 561
 562       /* Create list elts */
 563       pool_get (tsm->list_pool, per_user_translation_list_elt);
 564       clib_dlist_init (tsm->list_pool,
 565                        per_user_translation_list_elt - tsm->list_pool);
 566
 567       per_user_translation_list_elt->value = s - tsm->sessions;
 568       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 569       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 570
 571       clib_dlist_addtail (tsm->list_pool,
 572                           s->per_user_list_head_index,
 573                           per_user_translation_list_elt - tsm->list_pool);
 574
 575       s->user_index = u - tsm->users;
 576       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 577                                pool_elts (tsm->sessions));
 578     }
 579
 580   s->ha_last_refreshed = now;
 581
 582   return s;
 583 }
 584
 585 snat_session_t *
 586 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 587                       f64 now)
 588 {
 589   snat_session_t *s;
 590   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 591
 592   dlist_elt_t *oldest_elt;
 593   u64 sess_timeout_time;
 594   u32 oldest_index;
 595
 596   // no sessions
 597   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 598     goto alloc_new;
 599
 600   // no free sessions
 601   if (PREDICT_FALSE
 602       ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user))
 603     {
 604       if (nat44_max_translations_per_user_cleanup (u, thread_index, now))
 605         goto alloc_new;
 606
 607       nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 608                      clib_net_to_host_u32 (u->addr.as_u32));
 609       snat_ipfix_logging_max_entries_per_user (thread_index,
 610                                                sm->max_translations_per_user,
 611                                                u->addr.as_u32);
 612       return 0;
 613     }
 614
 615   oldest_index =
 616     clib_dlist_remove_head (tsm->list_pool,
 617                             u->sessions_per_user_list_head_index);
 618   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 619   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 620
 621   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 622   if (now >= sess_timeout_time)
 623     {
 624       // reuse old session
 625       clib_dlist_addtail (tsm->list_pool,
 626                           u->sessions_per_user_list_head_index, oldest_index);
 627       s = nat44_session_reuse_old (sm, u, s, thread_index, now);
 628     }
 629   else
 630     {
 631       // alloc new session
 632       clib_dlist_addhead (tsm->list_pool,
 633                           u->sessions_per_user_list_head_index, oldest_index);
 634     alloc_new:
 635       s = nat44_session_alloc_new (tsm, u, now);
 636       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 637                                pool_elts (tsm->sessions));
 638     }
 639   return s;
 640 }
 641
 642 void
 643 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 644                           int is_add)
 645 {
 646   fib_prefix_t prefix = {
 647     .fp_len = p_len,
 648     .fp_proto = FIB_PROTOCOL_IP4,
 649     .fp_addr = {
 650                 .ip4.as_u32 = addr->as_u32,
 651                 },
 652   };
 653   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 654
 655   if (is_add)
 656     fib_table_entry_update_one_path (fib_index,
 657                                      &prefix,
 658                                      nat_fib_src_low,
 659                                      (FIB_ENTRY_FLAG_CONNECTED |
 660                                       FIB_ENTRY_FLAG_LOCAL |
 661                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 662                                      DPO_PROTO_IP4,
 663                                      NULL,
 664                                      sw_if_index,
 665                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 666   else
 667     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 668 }
 669
 670 int
 671 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 672                   u8 twice_nat)
 673 {
 674   snat_address_t *ap;
 675   snat_interface_t *i;
 676   vlib_thread_main_t *tm = vlib_get_thread_main ();
 677
 678   if (twice_nat && !sm->endpoint_dependent)
 679     return VNET_API_ERROR_FEATURE_DISABLED;
 680
 681   /* Check if address already exists */
 682   /* *INDENT-OFF* */
 683   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 684     {
 685       if (ap->addr.as_u32 == addr->as_u32)
 686         return VNET_API_ERROR_VALUE_EXIST;
 687     }
 688   /* *INDENT-ON* */
 689
 690   if (twice_nat)
 691     vec_add2 (sm->twice_nat_addresses, ap, 1);
 692   else
 693     vec_add2 (sm->addresses, ap, 1);
 694
 695   ap->addr = *addr;
 696   if (vrf_id != ~0)
 697     ap->fib_index =
 698       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 699                                          nat_fib_src_low);
 700   else
 701     ap->fib_index = ~0;
 702 #define _(N, i, n, s) \
 703   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 704   ap->busy_##n##_ports = 0; \
 705   ap->busy_##n##_ports_per_thread = 0;\
 706   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 707   foreach_snat_protocol
 708 #undef _
 709     if (twice_nat)
 710     return 0;
 711
 712   /* Add external address to FIB */
 713   /* *INDENT-OFF* */
 714   pool_foreach (i, sm->interfaces,
 715   ({
 716     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 717       continue;
 718
 719     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 720     break;
 721   }));
 722   pool_foreach (i, sm->output_feature_interfaces,
 723   ({
 724     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 725       continue;
 726
 727     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 728     break;
 729   }));
 730   /* *INDENT-ON* */
 731
 732   return 0;
 733 }
 734
 735 static int
 736 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 737 {
 738   snat_static_mapping_t *m;
 739   /* *INDENT-OFF* */
 740   pool_foreach (m, sm->static_mappings,
 741   ({
 742       if (is_addr_only_static_mapping (m) ||
 743           is_out2in_only_static_mapping (m) ||
 744           is_identity_static_mapping (m))
 745         continue;
 746       if (m->external_addr.as_u32 == addr.as_u32)
 747         return 1;
 748   }));
 749   /* *INDENT-ON* */
 750
 751   return 0;
 752 }
 753
 754 static void
 755 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 756                                        ip4_address_t l_addr,
 757                                        u16 l_port,
 758                                        u32 sw_if_index,
 759                                        u16 e_port,
 760                                        u32 vrf_id,
 761                                        snat_protocol_t proto,
 762                                        int addr_only, int is_add, u8 * tag,
 763                                        int twice_nat, int out2in_only,
 764                                        int identity_nat)
 765 {
 766   snat_static_map_resolve_t *rp;
 767
 768   vec_add2 (sm->to_resolve, rp, 1);
 769   rp->l_addr.as_u32 = l_addr.as_u32;
 770   rp->l_port = l_port;
 771   rp->sw_if_index = sw_if_index;
 772   rp->e_port = e_port;
 773   rp->vrf_id = vrf_id;
 774   rp->proto = proto;
 775   rp->addr_only = addr_only;
 776   rp->is_add = is_add;
 777   rp->twice_nat = twice_nat;
 778   rp->out2in_only = out2in_only;
 779   rp->identity_nat = identity_nat;
 780   rp->tag = vec_dup (tag);
 781 }
 782
 783 static u32
 784 get_thread_idx_by_port (u16 e_port)
 785 {
 786   snat_main_t *sm = &snat_main;
 787   u32 thread_idx = sm->num_workers;
 788   if (sm->num_workers > 1)
 789     {
 790       thread_idx =
 791         sm->first_worker_index +
 792         sm->workers[(e_port - 1024) / sm->port_per_thread];
 793     }
 794   return thread_idx;
 795 }
 796
 797 int
 798 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 799                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 800                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 801                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 802                          u8 identity_nat)
 803 {
 804   snat_main_t *sm = &snat_main;
 805   snat_static_mapping_t *m;
 806   snat_session_key_t m_key;
 807   clib_bihash_kv_8_8_t kv, value;
 808   snat_address_t *a = 0;
 809   u32 fib_index = ~0;
 810   snat_interface_t *interface;
 811   int i;
 812   snat_main_per_thread_data_t *tsm;
 813   snat_user_key_t u_key;
 814   snat_user_t *u;
 815   dlist_elt_t *head, *elt;
 816   u32 elt_index, head_index;
 817   u32 ses_index;
 818   u64 user_index;
 819   snat_session_t *s;
 820   snat_static_map_resolve_t *rp, *rp_match = 0;
 821   nat44_lb_addr_port_t *local;
 822   u32 find = ~0;
 823
 824   if (!sm->endpoint_dependent)
 825     {
 826       if (twice_nat || out2in_only)
 827         return VNET_API_ERROR_FEATURE_DISABLED;
 828     }
 829
 830   /* If the external address is a specific interface address */
 831   if (sw_if_index != ~0)
 832     {
 833       ip4_address_t *first_int_addr;
 834
 835       for (i = 0; i < vec_len (sm->to_resolve); i++)
 836         {
 837           rp = sm->to_resolve + i;
 838           if (rp->sw_if_index != sw_if_index ||
 839               rp->l_addr.as_u32 != l_addr.as_u32 ||
 840               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 841             continue;
 842
 843           if (!addr_only)
 844             {
 845               if ((rp->l_port != l_port && rp->e_port != e_port)
 846                   || rp->proto != proto)
 847                 continue;
 848             }
 849
 850           rp_match = rp;
 851           break;
 852         }
 853
 854       /* Might be already set... */
 855       first_int_addr = ip4_interface_first_address
 856         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 857
 858       if (is_add)
 859         {
 860           if (rp_match)
 861             return VNET_API_ERROR_VALUE_EXIST;
 862
 863           snat_add_static_mapping_when_resolved
 864             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 865              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 866
 867           /* DHCP resolution required? */
 868           if (first_int_addr == 0)
 869             {
 870               return 0;
 871             }
 872           else
 873             {
 874               e_addr.as_u32 = first_int_addr->as_u32;
 875               /* Identity mapping? */
 876               if (l_addr.as_u32 == 0)
 877                 l_addr.as_u32 = e_addr.as_u32;
 878             }
 879         }
 880       else
 881         {
 882           if (!rp_match)
 883             return VNET_API_ERROR_NO_SUCH_ENTRY;
 884
 885           vec_del1 (sm->to_resolve, i);
 886
 887           if (first_int_addr)
 888             {
 889               e_addr.as_u32 = first_int_addr->as_u32;
 890               /* Identity mapping? */
 891               if (l_addr.as_u32 == 0)
 892                 l_addr.as_u32 = e_addr.as_u32;
 893             }
 894           else
 895             return 0;
 896         }
 897     }
 898
 899   m_key.addr = e_addr;
 900   m_key.port = addr_only ? 0 : e_port;
 901   m_key.protocol = addr_only ? 0 : proto;
 902   m_key.fib_index = 0;
 903   kv.key = m_key.as_u64;
 904   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 905     m = 0;
 906   else
 907     m = pool_elt_at_index (sm->static_mappings, value.value);
 908
 909   if (is_add)
 910     {
 911       if (m)
 912         {
 913           if (is_identity_static_mapping (m))
 914             {
 915               /* *INDENT-OFF* */
 916               pool_foreach (local, m->locals,
 917               ({
 918                 if (local->vrf_id == vrf_id)
 919                   return VNET_API_ERROR_VALUE_EXIST;
 920               }));
 921               /* *INDENT-ON* */
 922               pool_get (m->locals, local);
 923               local->vrf_id = vrf_id;
 924               local->fib_index =
 925                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 926                                                    nat_fib_src_low);
 927               m_key.addr = m->local_addr;
 928               m_key.port = m->local_port;
 929               m_key.protocol = m->proto;
 930               m_key.fib_index = local->fib_index;
 931               kv.key = m_key.as_u64;
 932               kv.value = m - sm->static_mappings;
 933               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 934               return 0;
 935             }
 936           else
 937             return VNET_API_ERROR_VALUE_EXIST;
 938         }
 939
 940       if (twice_nat && addr_only)
 941         return VNET_API_ERROR_UNSUPPORTED;
 942
 943       /* Convert VRF id to FIB index */
 944       if (vrf_id != ~0)
 945         fib_index =
 946           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 947                                              nat_fib_src_low);
 948       /* If not specified use inside VRF id from SNAT plugin startup config */
 949       else
 950         {
 951           fib_index = sm->inside_fib_index;
 952           vrf_id = sm->inside_vrf_id;
 953           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 954         }
 955
 956       if (!(out2in_only || identity_nat))
 957         {
 958           m_key.addr = l_addr;
 959           m_key.port = addr_only ? 0 : l_port;
 960           m_key.protocol = addr_only ? 0 : proto;
 961           m_key.fib_index = fib_index;
 962           kv.key = m_key.as_u64;
 963           if (!clib_bihash_search_8_8
 964               (&sm->static_mapping_by_local, &kv, &value))
 965             return VNET_API_ERROR_VALUE_EXIST;
 966         }
 967
 968       /* Find external address in allocated addresses and reserve port for
 969          address and port pair mapping when dynamic translations enabled */
 970       if (!(addr_only || sm->static_mapping_only || out2in_only))
 971         {
 972           for (i = 0; i < vec_len (sm->addresses); i++)
 973             {
 974               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 975                 {
 976                   a = sm->addresses + i;
 977                   /* External port must be unused */
 978                   switch (proto)
 979                     {
 980 #define _(N, j, n, s) \
 981                     case SNAT_PROTOCOL_##N: \
 982                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 983                         return VNET_API_ERROR_INVALID_VALUE; \
 984                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 985                       if (e_port > 1024) \
 986                         { \
 987                           a->busy_##n##_ports++; \
 988                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 989                         } \
 990                       break;
 991                       foreach_snat_protocol
 992 #undef _
 993                     default:
 994                       nat_elog_info ("unknown protocol");
 995                       return VNET_API_ERROR_INVALID_VALUE_2;
 996                     }
 997                   break;
 998                 }
 999             }
1000           /* External address must be allocated */
1001           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1002             {
1003               if (sw_if_index != ~0)
1004                 {
1005                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1006                     {
1007                       rp = sm->to_resolve + i;
1008                       if (rp->addr_only)
1009                         continue;
1010                       if (rp->sw_if_index != sw_if_index &&
1011                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1012                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1013                           rp->e_port != e_port && rp->proto != proto)
1014                         continue;
1015
1016                       vec_del1 (sm->to_resolve, i);
1017                       break;
1018                     }
1019                 }
1020               return VNET_API_ERROR_NO_SUCH_ENTRY;
1021             }
1022         }
1023
1024       pool_get (sm->static_mappings, m);
1025       clib_memset (m, 0, sizeof (*m));
1026       m->tag = vec_dup (tag);
1027       m->local_addr = l_addr;
1028       m->external_addr = e_addr;
1029       m->twice_nat = twice_nat;
1030       if (out2in_only)
1031         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1032       if (addr_only)
1033         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1034       if (identity_nat)
1035         {
1036           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1037           pool_get (m->locals, local);
1038           local->vrf_id = vrf_id;
1039           local->fib_index = fib_index;
1040         }
1041       else
1042         {
1043           m->vrf_id = vrf_id;
1044           m->fib_index = fib_index;
1045         }
1046       if (!addr_only)
1047         {
1048           m->local_port = l_port;
1049           m->external_port = e_port;
1050           m->proto = proto;
1051         }
1052
1053       if (sm->num_workers > 1)
1054         {
1055           ip4_header_t ip = {
1056             .src_address = m->local_addr,
1057           };
1058           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1059           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1060         }
1061       else
1062         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1063
1064       m_key.addr = m->local_addr;
1065       m_key.port = m->local_port;
1066       m_key.protocol = m->proto;
1067       m_key.fib_index = fib_index;
1068       kv.key = m_key.as_u64;
1069       kv.value = m - sm->static_mappings;
1070       if (!out2in_only)
1071         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1072
1073       m_key.addr = m->external_addr;
1074       m_key.port = m->external_port;
1075       m_key.fib_index = 0;
1076       kv.key = m_key.as_u64;
1077       kv.value = m - sm->static_mappings;
1078       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1079
1080       /* Delete dynamic sessions matching local address (+ local port) */
1081       if (!(sm->static_mapping_only))
1082         {
1083           u_key.addr = m->local_addr;
1084           u_key.fib_index = m->fib_index;
1085           kv.key = u_key.as_u64;
1086           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1087             {
1088               user_index = value.value;
1089               u = pool_elt_at_index (tsm->users, user_index);
1090               if (u->nsessions)
1091                 {
1092                   head_index = u->sessions_per_user_list_head_index;
1093                   head = pool_elt_at_index (tsm->list_pool, head_index);
1094                   elt_index = head->next;
1095                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1096                   ses_index = elt->value;
1097                   while (ses_index != ~0)
1098                     {
1099                       s = pool_elt_at_index (tsm->sessions, ses_index);
1100                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1101                       ses_index = elt->value;
1102
1103                       if (snat_is_session_static (s))
1104                         continue;
1105
1106                       if (!addr_only
1107                           && (clib_net_to_host_u16 (s->in2out.port) !=
1108                               m->local_port))
1109                         continue;
1110
1111                       nat_free_session_data (sm, s,
1112                                              tsm - sm->per_thread_data, 0);
1113                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1114
1115                       if (!addr_only && !sm->endpoint_dependent)
1116                         break;
1117                     }
1118                 }
1119             }
1120         }
1121     }
1122   else
1123     {
1124       if (!m)
1125         {
1126           if (sw_if_index != ~0)
1127             return 0;
1128           else
1129             return VNET_API_ERROR_NO_SUCH_ENTRY;
1130         }
1131
1132       if (identity_nat)
1133         {
1134           if (vrf_id == ~0)
1135             vrf_id = sm->inside_vrf_id;
1136
1137           /* *INDENT-OFF* */
1138           pool_foreach (local, m->locals,
1139           ({
1140             if (local->vrf_id == vrf_id)
1141               find = local - m->locals;
1142           }));
1143           /* *INDENT-ON* */
1144           if (find == ~0)
1145             return VNET_API_ERROR_NO_SUCH_ENTRY;
1146
1147           local = pool_elt_at_index (m->locals, find);
1148           fib_index = local->fib_index;
1149           pool_put (m->locals, local);
1150         }
1151       else
1152         fib_index = m->fib_index;
1153
1154       /* Free external address port */
1155       if (!(addr_only || sm->static_mapping_only || out2in_only))
1156         {
1157           for (i = 0; i < vec_len (sm->addresses); i++)
1158             {
1159               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1160                 {
1161                   a = sm->addresses + i;
1162                   switch (proto)
1163                     {
1164 #define _(N, j, n, s) \
1165                     case SNAT_PROTOCOL_##N: \
1166                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1167                       if (e_port > 1024) \
1168                         { \
1169                           a->busy_##n##_ports--; \
1170                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1171                         } \
1172                       break;
1173                       foreach_snat_protocol
1174 #undef _
1175                     default:
1176                       nat_elog_info ("unknown protocol");
1177                       return VNET_API_ERROR_INVALID_VALUE_2;
1178                     }
1179                   break;
1180                 }
1181             }
1182         }
1183
1184       if (sm->num_workers > 1)
1185         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1186       else
1187         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1188
1189       m_key.addr = m->local_addr;
1190       m_key.port = m->local_port;
1191       m_key.protocol = m->proto;
1192       m_key.fib_index = fib_index;
1193       kv.key = m_key.as_u64;
1194       if (!out2in_only)
1195         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1196
1197       /* Delete session(s) for static mapping if exist */
1198       if (!(sm->static_mapping_only) ||
1199           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1200         {
1201           u_key.addr = m->local_addr;
1202           u_key.fib_index = fib_index;
1203           kv.key = u_key.as_u64;
1204           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1205             {
1206               user_index = value.value;
1207               u = pool_elt_at_index (tsm->users, user_index);
1208               if (u->nstaticsessions)
1209                 {
1210                   head_index = u->sessions_per_user_list_head_index;
1211                   head = pool_elt_at_index (tsm->list_pool, head_index);
1212                   elt_index = head->next;
1213                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1214                   ses_index = elt->value;
1215                   while (ses_index != ~0)
1216                     {
1217                       s = pool_elt_at_index (tsm->sessions, ses_index);
1218                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1219                       ses_index = elt->value;
1220
1221                       if (!addr_only)
1222                         {
1223                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1224                               (clib_net_to_host_u16 (s->out2in.port) !=
1225                                e_port))
1226                             continue;
1227                         }
1228
1229                       if (is_lb_session (s))
1230                         continue;
1231
1232                       if (!snat_is_session_static (s))
1233                         continue;
1234
1235                       nat_free_session_data (sm, s,
1236                                              tsm - sm->per_thread_data, 0);
1237                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1238
1239                       if (!addr_only && !sm->endpoint_dependent)
1240                         break;
1241                     }
1242                 }
1243             }
1244         }
1245
1246       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1247       if (pool_elts (m->locals))
1248         return 0;
1249
1250       m_key.addr = m->external_addr;
1251       m_key.port = m->external_port;
1252       m_key.fib_index = 0;
1253       kv.key = m_key.as_u64;
1254       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1255
1256       vec_free (m->tag);
1257       vec_free (m->workers);
1258       /* Delete static mapping from pool */
1259       pool_put (sm->static_mappings, m);
1260     }
1261
1262   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1263     return 0;
1264
1265   /* Add/delete external address to FIB */
1266   /* *INDENT-OFF* */
1267   pool_foreach (interface, sm->interfaces,
1268   ({
1269     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1270       continue;
1271
1272     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1273     break;
1274   }));
1275   pool_foreach (interface, sm->output_feature_interfaces,
1276   ({
1277     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1278       continue;
1279
1280     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1281     break;
1282   }));
1283   /* *INDENT-ON* */
1284
1285   return 0;
1286 }
1287
1288 int
1289 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1290                                  snat_protocol_t proto,
1291                                  nat44_lb_addr_port_t * locals, u8 is_add,
1292                                  twice_nat_type_t twice_nat, u8 out2in_only,
1293                                  u8 * tag, u32 affinity)
1294 {
1295   snat_main_t *sm = &snat_main;
1296   snat_static_mapping_t *m;
1297   snat_session_key_t m_key;
1298   clib_bihash_kv_8_8_t kv, value;
1299   snat_address_t *a = 0;
1300   int i;
1301   nat44_lb_addr_port_t *local;
1302   u32 elt_index, head_index, ses_index;
1303   snat_main_per_thread_data_t *tsm;
1304   snat_user_key_t u_key;
1305   snat_user_t *u;
1306   snat_session_t *s;
1307   dlist_elt_t *head, *elt;
1308   uword *bitmap = 0;
1309
1310   if (!sm->endpoint_dependent)
1311     return VNET_API_ERROR_FEATURE_DISABLED;
1312
1313   m_key.addr = e_addr;
1314   m_key.port = e_port;
1315   m_key.protocol = proto;
1316   m_key.fib_index = 0;
1317   kv.key = m_key.as_u64;
1318   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1319     m = 0;
1320   else
1321     m = pool_elt_at_index (sm->static_mappings, value.value);
1322
1323   if (is_add)
1324     {
1325       if (m)
1326         return VNET_API_ERROR_VALUE_EXIST;
1327
1328       if (vec_len (locals) < 2)
1329         return VNET_API_ERROR_INVALID_VALUE;
1330
1331       /* Find external address in allocated addresses and reserve port for
1332          address and port pair mapping when dynamic translations enabled */
1333       if (!(sm->static_mapping_only || out2in_only))
1334         {
1335           for (i = 0; i < vec_len (sm->addresses); i++)
1336             {
1337               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1338                 {
1339                   a = sm->addresses + i;
1340                   /* External port must be unused */
1341                   switch (proto)
1342                     {
1343 #define _(N, j, n, s) \
1344                     case SNAT_PROTOCOL_##N: \
1345                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1346                         return VNET_API_ERROR_INVALID_VALUE; \
1347                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1348                       if (e_port > 1024) \
1349                         { \
1350                           a->busy_##n##_ports++; \
1351                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1352                         } \
1353                       break;
1354                       foreach_snat_protocol
1355 #undef _
1356                     default:
1357                       nat_elog_info ("unknown protocol");
1358                       return VNET_API_ERROR_INVALID_VALUE_2;
1359                     }
1360                   break;
1361                 }
1362             }
1363           /* External address must be allocated */
1364           if (!a)
1365             return VNET_API_ERROR_NO_SUCH_ENTRY;
1366         }
1367
1368       pool_get (sm->static_mappings, m);
1369       clib_memset (m, 0, sizeof (*m));
1370       m->tag = vec_dup (tag);
1371       m->external_addr = e_addr;
1372       m->external_port = e_port;
1373       m->proto = proto;
1374       m->twice_nat = twice_nat;
1375       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1376       if (out2in_only)
1377         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1378       m->affinity = affinity;
1379
1380       if (affinity)
1381         m->affinity_per_service_list_head_index =
1382           nat_affinity_get_per_service_list_head_index ();
1383       else
1384         m->affinity_per_service_list_head_index = ~0;
1385
1386       m_key.addr = m->external_addr;
1387       m_key.port = m->external_port;
1388       m_key.protocol = m->proto;
1389       m_key.fib_index = 0;
1390       kv.key = m_key.as_u64;
1391       kv.value = m - sm->static_mappings;
1392       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1393         {
1394           nat_elog_err ("static_mapping_by_external key add failed");
1395           return VNET_API_ERROR_UNSPECIFIED;
1396         }
1397
1398       m_key.fib_index = m->fib_index;
1399       for (i = 0; i < vec_len (locals); i++)
1400         {
1401           locals[i].fib_index =
1402             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1403                                                locals[i].vrf_id,
1404                                                nat_fib_src_low);
1405           m_key.addr = locals[i].addr;
1406           m_key.fib_index = locals[i].fib_index;
1407           if (!out2in_only)
1408             {
1409               m_key.port = locals[i].port;
1410               kv.key = m_key.as_u64;
1411               kv.value = m - sm->static_mappings;
1412               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1413             }
1414           locals[i].prefix = (i == 0) ? locals[i].probability :
1415             (locals[i - 1].prefix + locals[i].probability);
1416           pool_get (m->locals, local);
1417           *local = locals[i];
1418           if (sm->num_workers > 1)
1419             {
1420               ip4_header_t ip = {
1421                 .src_address = locals[i].addr,
1422               };
1423               bitmap =
1424                 clib_bitmap_set (bitmap,
1425                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1426                                  1);
1427             }
1428         }
1429
1430       /* Assign workers */
1431       if (sm->num_workers > 1)
1432         {
1433           /* *INDENT-OFF* */
1434           clib_bitmap_foreach (i, bitmap,
1435             ({
1436                vec_add1(m->workers, i);
1437             }));
1438           /* *INDENT-ON* */
1439         }
1440     }
1441   else
1442     {
1443       if (!m)
1444         return VNET_API_ERROR_NO_SUCH_ENTRY;
1445
1446       if (!is_lb_static_mapping (m))
1447         return VNET_API_ERROR_INVALID_VALUE;
1448
1449       /* Free external address port */
1450       if (!(sm->static_mapping_only || out2in_only))
1451         {
1452           for (i = 0; i < vec_len (sm->addresses); i++)
1453             {
1454               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1455                 {
1456                   a = sm->addresses + i;
1457                   switch (proto)
1458                     {
1459 #define _(N, j, n, s) \
1460                     case SNAT_PROTOCOL_##N: \
1461                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1462                       if (e_port > 1024) \
1463                         { \
1464                           a->busy_##n##_ports--; \
1465                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1466                         } \
1467                       break;
1468                       foreach_snat_protocol
1469 #undef _
1470                     default:
1471                       nat_elog_info ("unknown protocol");
1472                       return VNET_API_ERROR_INVALID_VALUE_2;
1473                     }
1474                   break;
1475                 }
1476             }
1477         }
1478
1479       m_key.addr = m->external_addr;
1480       m_key.port = m->external_port;
1481       m_key.protocol = m->proto;
1482       m_key.fib_index = 0;
1483       kv.key = m_key.as_u64;
1484       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1485         {
1486           nat_elog_err ("static_mapping_by_external key del failed");
1487           return VNET_API_ERROR_UNSPECIFIED;
1488         }
1489
1490       /* *INDENT-OFF* */
1491       pool_foreach (local, m->locals,
1492       ({
1493           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1494                             nat_fib_src_low);
1495           m_key.addr = local->addr;
1496           if (!out2in_only)
1497             {
1498               m_key.port = local->port;
1499               m_key.fib_index = local->fib_index;
1500               kv.key = m_key.as_u64;
1501               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1502                 {
1503                   nat_elog_err ("static_mapping_by_local key del failed");
1504                   return VNET_API_ERROR_UNSPECIFIED;
1505                 }
1506             }
1507
1508           if (sm->num_workers > 1)
1509             {
1510               ip4_header_t ip = {
1511                 .src_address = local->addr,
1512               };
1513               tsm = vec_elt_at_index (sm->per_thread_data,
1514                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1515             }
1516           else
1517             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1518
1519           /* Delete sessions */
1520           u_key.addr = local->addr;
1521           u_key.fib_index = local->fib_index;
1522           kv.key = u_key.as_u64;
1523           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1524             {
1525               u = pool_elt_at_index (tsm->users, value.value);
1526               if (u->nstaticsessions)
1527                 {
1528                   head_index = u->sessions_per_user_list_head_index;
1529                   head = pool_elt_at_index (tsm->list_pool, head_index);
1530                   elt_index = head->next;
1531                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1532                   ses_index = elt->value;
1533                   while (ses_index != ~0)
1534                     {
1535                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1536                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1537                       ses_index = elt->value;
1538
1539                       if (!(is_lb_session (s)))
1540                         continue;
1541
1542                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1543                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1544                         continue;
1545
1546                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1547                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1548                     }
1549                 }
1550             }
1551       }));
1552       /* *INDENT-ON* */
1553       if (m->affinity)
1554         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1555       pool_free (m->locals);
1556       vec_free (m->tag);
1557       vec_free (m->workers);
1558
1559       pool_put (sm->static_mappings, m);
1560     }
1561
1562   return 0;
1563 }
1564
1565 int
1566 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1567                                        ip4_address_t l_addr, u16 l_port,
1568                                        snat_protocol_t proto, u32 vrf_id,
1569                                        u8 probability, u8 is_add)
1570 {
1571   snat_main_t *sm = &snat_main;
1572   snat_static_mapping_t *m = 0;
1573   snat_session_key_t m_key;
1574   clib_bihash_kv_8_8_t kv, value;
1575   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1576   snat_main_per_thread_data_t *tsm;
1577   snat_user_key_t u_key;
1578   snat_user_t *u;
1579   snat_session_t *s;
1580   dlist_elt_t *head, *elt;
1581   u32 elt_index, head_index, ses_index, *locals = 0;
1582   uword *bitmap = 0;
1583   int i;
1584
1585   if (!sm->endpoint_dependent)
1586     return VNET_API_ERROR_FEATURE_DISABLED;
1587
1588   m_key.addr = e_addr;
1589   m_key.port = e_port;
1590   m_key.protocol = proto;
1591   m_key.fib_index = 0;
1592   kv.key = m_key.as_u64;
1593   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1594     m = pool_elt_at_index (sm->static_mappings, value.value);
1595
1596   if (!m)
1597     return VNET_API_ERROR_NO_SUCH_ENTRY;
1598
1599   if (!is_lb_static_mapping (m))
1600     return VNET_API_ERROR_INVALID_VALUE;
1601
1602   /* *INDENT-OFF* */
1603   pool_foreach (local, m->locals,
1604   ({
1605     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1606         (local->vrf_id == vrf_id))
1607       {
1608         match_local = local;
1609         break;
1610       }
1611   }));
1612   /* *INDENT-ON* */
1613
1614   if (is_add)
1615     {
1616       if (match_local)
1617         return VNET_API_ERROR_VALUE_EXIST;
1618
1619       pool_get (m->locals, local);
1620       clib_memset (local, 0, sizeof (*local));
1621       local->addr.as_u32 = l_addr.as_u32;
1622       local->port = l_port;
1623       local->probability = probability;
1624       local->vrf_id = vrf_id;
1625       local->fib_index =
1626         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1627                                            nat_fib_src_low);
1628
1629       if (!is_out2in_only_static_mapping (m))
1630         {
1631           m_key.addr = l_addr;
1632           m_key.port = l_port;
1633           m_key.fib_index = local->fib_index;
1634           kv.key = m_key.as_u64;
1635           kv.value = m - sm->static_mappings;
1636           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1637             nat_elog_err ("static_mapping_by_local key add failed");
1638         }
1639     }
1640   else
1641     {
1642       if (!match_local)
1643         return VNET_API_ERROR_NO_SUCH_ENTRY;
1644
1645       if (pool_elts (m->locals) < 3)
1646         return VNET_API_ERROR_UNSPECIFIED;
1647
1648       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1649                         nat_fib_src_low);
1650
1651       if (!is_out2in_only_static_mapping (m))
1652         {
1653           m_key.addr = l_addr;
1654           m_key.port = l_port;
1655           m_key.fib_index = match_local->fib_index;
1656           kv.key = m_key.as_u64;
1657           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1658             nat_elog_err ("static_mapping_by_local key del failed");
1659         }
1660
1661       if (sm->num_workers > 1)
1662         {
1663           ip4_header_t ip = {
1664             .src_address = local->addr,
1665           };
1666           tsm = vec_elt_at_index (sm->per_thread_data,
1667                                   sm->worker_in2out_cb (&ip, m->fib_index,
1668                                                         0));
1669         }
1670       else
1671         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1672
1673       /* Delete sessions */
1674       u_key.addr = match_local->addr;
1675       u_key.fib_index = match_local->fib_index;
1676       kv.key = u_key.as_u64;
1677       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1678         {
1679           u = pool_elt_at_index (tsm->users, value.value);
1680           if (u->nstaticsessions)
1681             {
1682               head_index = u->sessions_per_user_list_head_index;
1683               head = pool_elt_at_index (tsm->list_pool, head_index);
1684               elt_index = head->next;
1685               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1686               ses_index = elt->value;
1687               while (ses_index != ~0)
1688                 {
1689                   s = pool_elt_at_index (tsm->sessions, ses_index);
1690                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1691                   ses_index = elt->value;
1692
1693                   if (!(is_lb_session (s)))
1694                     continue;
1695
1696                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1697                       (clib_net_to_host_u16 (s->in2out.port) !=
1698                        match_local->port))
1699                     continue;
1700
1701                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1702                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1703                 }
1704             }
1705         }
1706
1707       pool_put (m->locals, match_local);
1708     }
1709
1710   vec_free (m->workers);
1711
1712   /* *INDENT-OFF* */
1713   pool_foreach (local, m->locals,
1714   ({
1715     vec_add1 (locals, local - m->locals);
1716     if (sm->num_workers > 1)
1717       {
1718         ip4_header_t ip;
1719         ip.src_address.as_u32 = local->addr.as_u32,
1720         bitmap = clib_bitmap_set (bitmap,
1721                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1722                                   1);
1723       }
1724   }));
1725   /* *INDENT-ON* */
1726
1727   ASSERT (vec_len (locals) > 1);
1728
1729   local = pool_elt_at_index (m->locals, locals[0]);
1730   local->prefix = local->probability;
1731   for (i = 1; i < vec_len (locals); i++)
1732     {
1733       local = pool_elt_at_index (m->locals, locals[i]);
1734       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1735       local->prefix = local->probability + prev_local->prefix;
1736     }
1737
1738   /* Assign workers */
1739   if (sm->num_workers > 1)
1740     {
1741       /* *INDENT-OFF* */
1742       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1743       /* *INDENT-ON* */
1744     }
1745
1746   return 0;
1747 }
1748
1749 int
1750 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1751                   u8 twice_nat)
1752 {
1753   snat_address_t *a = 0;
1754   snat_session_t *ses;
1755   u32 *ses_to_be_removed = 0, *ses_index;
1756   snat_main_per_thread_data_t *tsm;
1757   snat_static_mapping_t *m;
1758   snat_interface_t *interface;
1759   int i;
1760   snat_address_t *addresses =
1761     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1762
1763   /* Find SNAT address */
1764   for (i = 0; i < vec_len (addresses); i++)
1765     {
1766       if (addresses[i].addr.as_u32 == addr.as_u32)
1767         {
1768           a = addresses + i;
1769           break;
1770         }
1771     }
1772   if (!a)
1773     return VNET_API_ERROR_NO_SUCH_ENTRY;
1774
1775   if (delete_sm)
1776     {
1777       /* *INDENT-OFF* */
1778       pool_foreach (m, sm->static_mappings,
1779       ({
1780           if (m->external_addr.as_u32 == addr.as_u32)
1781             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1782                                             m->local_port, m->external_port,
1783                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1784                                             m->proto, 0, m->twice_nat,
1785                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1786       }));
1787       /* *INDENT-ON* */
1788     }
1789   else
1790     {
1791       /* Check if address is used in some static mapping */
1792       if (is_snat_address_used_in_static_mapping (sm, addr))
1793         {
1794           nat_elog_notice ("address used in static mapping");
1795           return VNET_API_ERROR_UNSPECIFIED;
1796         }
1797     }
1798
1799   if (a->fib_index != ~0)
1800     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1801
1802   /* Delete sessions using address */
1803   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1804     {
1805       /* *INDENT-OFF* */
1806       vec_foreach (tsm, sm->per_thread_data)
1807         {
1808           pool_foreach (ses, tsm->sessions, ({
1809             if (ses->out2in.addr.as_u32 == addr.as_u32)
1810               {
1811                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1812                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1813               }
1814           }));
1815
1816           vec_foreach (ses_index, ses_to_be_removed)
1817             {
1818               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1819               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1820             }
1821
1822           vec_free (ses_to_be_removed);
1823         }
1824       /* *INDENT-ON* */
1825     }
1826
1827 #define _(N, i, n, s) \
1828   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1829   vec_free (a->busy_##n##_ports_per_thread);
1830   foreach_snat_protocol
1831 #undef _
1832     if (twice_nat)
1833     {
1834       vec_del1 (sm->twice_nat_addresses, i);
1835       return 0;
1836     }
1837   else
1838     vec_del1 (sm->addresses, i);
1839
1840   /* Delete external address from FIB */
1841   /* *INDENT-OFF* */
1842   pool_foreach (interface, sm->interfaces,
1843   ({
1844     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1845       continue;
1846
1847     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1848     break;
1849   }));
1850   pool_foreach (interface, sm->output_feature_interfaces,
1851   ({
1852     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1853       continue;
1854
1855     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1856     break;
1857   }));
1858   /* *INDENT-ON* */
1859
1860   return 0;
1861 }
1862
1863 int
1864 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1865 {
1866   snat_main_t *sm = &snat_main;
1867   snat_interface_t *i;
1868   const char *feature_name, *del_feature_name;
1869   snat_address_t *ap;
1870   snat_static_mapping_t *m;
1871   snat_det_map_t *dm;
1872   nat_outside_fib_t *outside_fib;
1873   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1874                                                        sw_if_index);
1875
1876   if (sm->out2in_dpo && !is_inside)
1877     return VNET_API_ERROR_UNSUPPORTED;
1878
1879   /* *INDENT-OFF* */
1880   pool_foreach (i, sm->output_feature_interfaces,
1881   ({
1882     if (i->sw_if_index == sw_if_index)
1883       return VNET_API_ERROR_VALUE_EXIST;
1884   }));
1885   /* *INDENT-ON* */
1886
1887   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1888     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1889   else
1890     {
1891       if (sm->num_workers > 1 && !sm->deterministic)
1892         feature_name =
1893           is_inside ? "nat44-in2out-worker-handoff" :
1894           "nat44-out2in-worker-handoff";
1895       else if (sm->deterministic)
1896         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1897       else if (sm->endpoint_dependent)
1898         {
1899           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1900         }
1901       else
1902         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1903     }
1904
1905   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1906     sm->fq_in2out_index =
1907       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1908
1909   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1910     sm->fq_out2in_index =
1911       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1912
1913   if (!is_inside)
1914     {
1915       /* *INDENT-OFF* */
1916       vec_foreach (outside_fib, sm->outside_fibs)
1917         {
1918           if (outside_fib->fib_index == fib_index)
1919             {
1920               if (is_del)
1921                 {
1922                   outside_fib->refcount--;
1923                   if (!outside_fib->refcount)
1924                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1925                 }
1926               else
1927                 outside_fib->refcount++;
1928               goto feature_set;
1929             }
1930         }
1931       /* *INDENT-ON* */
1932       if (!is_del)
1933         {
1934           vec_add2 (sm->outside_fibs, outside_fib, 1);
1935           outside_fib->refcount = 1;
1936           outside_fib->fib_index = fib_index;
1937         }
1938     }
1939 feature_set:
1940   /* *INDENT-OFF* */
1941   pool_foreach (i, sm->interfaces,
1942   ({
1943     if (i->sw_if_index == sw_if_index)
1944       {
1945         if (is_del)
1946           {
1947             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1948               {
1949                 if (is_inside)
1950                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1951                 else
1952                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1953
1954                 if (sm->num_workers > 1 && !sm->deterministic)
1955                   {
1956                     del_feature_name = "nat44-handoff-classify";
1957                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1958                                                  "nat44-out2in-worker-handoff";
1959                   }
1960                 else if (sm->deterministic)
1961                   {
1962                     del_feature_name = "nat44-det-classify";
1963                     feature_name = !is_inside ?  "nat44-det-in2out" :
1964                                                  "nat44-det-out2in";
1965                   }
1966                 else if (sm->endpoint_dependent)
1967                   {
1968                     del_feature_name = "nat44-ed-classify";
1969                     feature_name = !is_inside ?  "nat-pre-in2out" :
1970                                                  "nat-pre-out2in";
1971                   }
1972                 else
1973                   {
1974                     del_feature_name = "nat44-classify";
1975                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1976                   }
1977
1978                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1979                 if (rv)
1980                   return rv;
1981                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1982                                              sw_if_index, 0, 0, 0);
1983                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1984                                              sw_if_index, 1, 0, 0);
1985                 if (!is_inside)
1986                   {
1987                     if (sm->endpoint_dependent)
1988                       vnet_feature_enable_disable ("ip4-local",
1989                                                    "nat44-ed-hairpinning",
1990                                                    sw_if_index, 1, 0, 0);
1991                     else if (!sm->deterministic)
1992                       vnet_feature_enable_disable ("ip4-local",
1993                                                    "nat44-hairpinning",
1994                                                    sw_if_index, 1, 0, 0);
1995                   }
1996               }
1997             else
1998               {
1999                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2000                 if (rv)
2001                   return rv;
2002                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2003                                              sw_if_index, 0, 0, 0);
2004                 pool_put (sm->interfaces, i);
2005                 if (is_inside)
2006                   {
2007                     if (sm->endpoint_dependent)
2008                       vnet_feature_enable_disable ("ip4-local",
2009                                                    "nat44-ed-hairpinning",
2010                                                    sw_if_index, 0, 0, 0);
2011                     else if (!sm->deterministic)
2012                       vnet_feature_enable_disable ("ip4-local",
2013                                                    "nat44-hairpinning",
2014                                                    sw_if_index, 0, 0, 0);
2015                   }
2016               }
2017           }
2018         else
2019           {
2020             if ((nat_interface_is_inside(i) && is_inside) ||
2021                 (nat_interface_is_outside(i) && !is_inside))
2022               return 0;
2023
2024             if (sm->num_workers > 1 && !sm->deterministic)
2025               {
2026                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2027                                                  "nat44-out2in-worker-handoff";
2028                 feature_name = "nat44-handoff-classify";
2029               }
2030             else if (sm->deterministic)
2031               {
2032                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
2033                                                  "nat44-det-out2in";
2034                 feature_name = "nat44-det-classify";
2035               }
2036             else if (sm->endpoint_dependent)
2037               {
2038                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2039                                                  "nat-pre-out2in";
2040
2041                 feature_name = "nat44-ed-classify";
2042               }
2043             else
2044               {
2045                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2046                 feature_name = "nat44-classify";
2047               }
2048
2049             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2050             if (rv)
2051               return rv;
2052             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2053                                          sw_if_index, 0, 0, 0);
2054             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2055                                          sw_if_index, 1, 0, 0);
2056             if (!is_inside)
2057               {
2058                 if (sm->endpoint_dependent)
2059                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2060                                                sw_if_index, 0, 0, 0);
2061                 else if (!sm->deterministic)
2062                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2063                                                sw_if_index, 0, 0, 0);
2064               }
2065             goto set_flags;
2066           }
2067
2068         goto fib;
2069       }
2070   }));
2071   /* *INDENT-ON* */
2072
2073   if (is_del)
2074     return VNET_API_ERROR_NO_SUCH_ENTRY;
2075
2076   pool_get (sm->interfaces, i);
2077   i->sw_if_index = sw_if_index;
2078   i->flags = 0;
2079   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2080                                0);
2081
2082   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2083   if (rv)
2084     return rv;
2085
2086   if (is_inside && !sm->out2in_dpo)
2087     {
2088       if (sm->endpoint_dependent)
2089         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2090                                      sw_if_index, 1, 0, 0);
2091       else if (!sm->deterministic)
2092         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2093                                      sw_if_index, 1, 0, 0);
2094     }
2095
2096 set_flags:
2097   if (is_inside)
2098     {
2099       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2100       return 0;
2101     }
2102   else
2103     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2104
2105   /* Add/delete external addresses to FIB */
2106 fib:
2107   /* *INDENT-OFF* */
2108   vec_foreach (ap, sm->addresses)
2109     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2110
2111   pool_foreach (m, sm->static_mappings,
2112   ({
2113     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2114       continue;
2115
2116     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2117   }));
2118
2119   pool_foreach (dm, sm->det_maps,
2120   ({
2121     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2122   }));
2123   /* *INDENT-ON* */
2124
2125   return 0;
2126 }
2127
2128 int
2129 snat_interface_add_del_output_feature (u32 sw_if_index,
2130                                        u8 is_inside, int is_del)
2131 {
2132   snat_main_t *sm = &snat_main;
2133   snat_interface_t *i;
2134   snat_address_t *ap;
2135   snat_static_mapping_t *m;
2136   nat_outside_fib_t *outside_fib;
2137   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2138                                                        sw_if_index);
2139
2140
2141   if (sm->deterministic ||
2142       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2143     return VNET_API_ERROR_UNSUPPORTED;
2144
2145   /* *INDENT-OFF* */
2146   pool_foreach (i, sm->interfaces,
2147   ({
2148     if (i->sw_if_index == sw_if_index)
2149       return VNET_API_ERROR_VALUE_EXIST;
2150   }));
2151   /* *INDENT-ON* */
2152
2153   if (!is_inside)
2154     {
2155       /* *INDENT-OFF* */
2156       vec_foreach (outside_fib, sm->outside_fibs)
2157         {
2158           if (outside_fib->fib_index == fib_index)
2159             {
2160               if (is_del)
2161                 {
2162                   outside_fib->refcount--;
2163                   if (!outside_fib->refcount)
2164                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2165                 }
2166               else
2167                 outside_fib->refcount++;
2168               goto feature_set;
2169             }
2170         }
2171       /* *INDENT-ON* */
2172       if (!is_del)
2173         {
2174           vec_add2 (sm->outside_fibs, outside_fib, 1);
2175           outside_fib->refcount = 1;
2176           outside_fib->fib_index = fib_index;
2177         }
2178     }
2179
2180 feature_set:
2181   if (is_inside)
2182     {
2183       if (sm->endpoint_dependent)
2184         {
2185           int rv =
2186             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2187           if (rv)
2188             return rv;
2189           rv =
2190             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2191                                                             !is_del);
2192           if (rv)
2193             return rv;
2194           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2195                                        sw_if_index, !is_del, 0, 0);
2196           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2197                                        sw_if_index, !is_del, 0, 0);
2198         }
2199       else
2200         {
2201           int rv =
2202             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2203           if (rv)
2204             return rv;
2205           rv =
2206             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2207                                                             !is_del);
2208           if (rv)
2209             return rv;
2210           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2211                                        sw_if_index, !is_del, 0, 0);
2212           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2213                                        sw_if_index, !is_del, 0, 0);
2214         }
2215       goto fq;
2216     }
2217
2218   if (sm->num_workers > 1)
2219     {
2220       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2221       if (rv)
2222         return rv;
2223       rv =
2224         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2225       if (rv)
2226         return rv;
2227       vnet_feature_enable_disable ("ip4-unicast",
2228                                    "nat44-out2in-worker-handoff",
2229                                    sw_if_index, !is_del, 0, 0);
2230       vnet_feature_enable_disable ("ip4-output",
2231                                    "nat44-in2out-output-worker-handoff",
2232                                    sw_if_index, !is_del, 0, 0);
2233     }
2234   else
2235     {
2236       if (sm->endpoint_dependent)
2237         {
2238           int rv =
2239             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2240           if (rv)
2241             return rv;
2242           rv =
2243             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2244                                                             !is_del);
2245           if (rv)
2246             return rv;
2247           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2248                                        sw_if_index, !is_del, 0, 0);
2249           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2250                                        sw_if_index, !is_del, 0, 0);
2251         }
2252       else
2253         {
2254           int rv =
2255             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2256           if (rv)
2257             return rv;
2258           rv =
2259             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2260                                                             !is_del);
2261           if (rv)
2262             return rv;
2263           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2264                                        sw_if_index, !is_del, 0, 0);
2265           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2266                                        sw_if_index, !is_del, 0, 0);
2267         }
2268     }
2269
2270 fq:
2271   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2272     sm->fq_in2out_output_index =
2273       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2274
2275   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2276     sm->fq_out2in_index =
2277       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2278
2279   /* *INDENT-OFF* */
2280   pool_foreach (i, sm->output_feature_interfaces,
2281   ({
2282     if (i->sw_if_index == sw_if_index)
2283       {
2284         if (is_del)
2285           pool_put (sm->output_feature_interfaces, i);
2286         else
2287           return VNET_API_ERROR_VALUE_EXIST;
2288
2289         goto fib;
2290       }
2291   }));
2292   /* *INDENT-ON* */
2293
2294   if (is_del)
2295     return VNET_API_ERROR_NO_SUCH_ENTRY;
2296
2297   pool_get (sm->output_feature_interfaces, i);
2298   i->sw_if_index = sw_if_index;
2299   i->flags = 0;
2300   if (is_inside)
2301     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2302   else
2303     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2304
2305   /* Add/delete external addresses to FIB */
2306 fib:
2307   if (is_inside)
2308     return 0;
2309
2310   /* *INDENT-OFF* */
2311   vec_foreach (ap, sm->addresses)
2312     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2313
2314   pool_foreach (m, sm->static_mappings,
2315   ({
2316     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2317       continue;
2318
2319     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2320   }));
2321   /* *INDENT-ON* */
2322
2323   return 0;
2324 }
2325
2326 int
2327 snat_set_workers (uword * bitmap)
2328 {
2329   snat_main_t *sm = &snat_main;
2330   int i, j = 0;
2331
2332   if (sm->num_workers < 2)
2333     return VNET_API_ERROR_FEATURE_DISABLED;
2334
2335   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2336     return VNET_API_ERROR_INVALID_WORKER;
2337
2338   vec_free (sm->workers);
2339   /* *INDENT-OFF* */
2340   clib_bitmap_foreach (i, bitmap,
2341     ({
2342       vec_add1(sm->workers, i);
2343       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2344       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2345       j++;
2346     }));
2347   /* *INDENT-ON* */
2348
2349   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2350   sm->num_snat_thread = _vec_len (sm->workers);
2351
2352   return 0;
2353 }
2354
2355 static void
2356 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2357                          u32 old_fib_index)
2358 {
2359   snat_main_t *sm = &snat_main;
2360   nat_outside_fib_t *outside_fib;
2361   snat_interface_t *i;
2362   u8 is_add = 1;
2363   u8 match = 0;
2364
2365   if (new_fib_index == old_fib_index)
2366     return;
2367
2368   if (!vec_len (sm->outside_fibs))
2369     return;
2370
2371   /* *INDENT-OFF* */
2372   pool_foreach (i, sm->interfaces,
2373     ({
2374       if (i->sw_if_index == sw_if_index)
2375         {
2376           if (!(nat_interface_is_outside (i)))
2377             return;
2378           match = 1;
2379         }
2380     }));
2381
2382   pool_foreach (i, sm->output_feature_interfaces,
2383     ({
2384       if (i->sw_if_index == sw_if_index)
2385         {
2386           if (!(nat_interface_is_outside (i)))
2387             return;
2388           match = 1;
2389         }
2390     }));
2391   /* *INDENT-ON* */
2392
2393   if (!match)
2394     return;
2395
2396   vec_foreach (outside_fib, sm->outside_fibs)
2397   {
2398     if (outside_fib->fib_index == old_fib_index)
2399       {
2400         outside_fib->refcount--;
2401         if (!outside_fib->refcount)
2402           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2403         break;
2404       }
2405   }
2406
2407   vec_foreach (outside_fib, sm->outside_fibs)
2408   {
2409     if (outside_fib->fib_index == new_fib_index)
2410       {
2411         outside_fib->refcount++;
2412         is_add = 0;
2413         break;
2414       }
2415   }
2416
2417   if (is_add)
2418     {
2419       vec_add2 (sm->outside_fibs, outside_fib, 1);
2420       outside_fib->refcount = 1;
2421       outside_fib->fib_index = new_fib_index;
2422     }
2423 }
2424
2425 static void
2426 snat_ip4_table_bind (ip4_main_t * im,
2427                      uword opaque,
2428                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2429 {
2430   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2431 }
2432
2433 static void
2434 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2435                                        uword opaque,
2436                                        u32 sw_if_index,
2437                                        ip4_address_t * address,
2438                                        u32 address_length,
2439                                        u32 if_address_index, u32 is_delete);
2440
2441 static void
2442 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2443                                  uword opaque,
2444                                  u32 sw_if_index,
2445                                  ip4_address_t * address,
2446                                  u32 address_length,
2447                                  u32 if_address_index, u32 is_delete);
2448
2449 static int
2450 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2451                                  u32 fib_index,
2452                                  u32 thread_index,
2453                                  snat_session_key_t * k,
2454                                  u16 port_per_thread, u32 snat_thread_index);
2455
2456 static clib_error_t *
2457 snat_init (vlib_main_t * vm)
2458 {
2459   snat_main_t *sm = &snat_main;
2460   clib_error_t *error = 0;
2461   ip4_main_t *im = &ip4_main;
2462   ip_lookup_main_t *lm = &im->lookup_main;
2463   uword *p;
2464   vlib_thread_registration_t *tr;
2465   vlib_thread_main_t *tm = vlib_get_thread_main ();
2466   uword *bitmap = 0;
2467   u32 i;
2468   ip4_add_del_interface_address_callback_t cb4;
2469   vlib_node_t *node;
2470
2471   sm->vlib_main = vm;
2472   sm->vnet_main = vnet_get_main ();
2473   sm->ip4_main = im;
2474   sm->ip4_lookup_main = lm;
2475   sm->api_main = vlibapi_get_main ();
2476   sm->first_worker_index = 0;
2477   sm->num_workers = 0;
2478   sm->num_snat_thread = 1;
2479   sm->workers = 0;
2480   sm->port_per_thread = 0xffff - 1024;
2481   sm->fq_in2out_index = ~0;
2482   sm->fq_in2out_output_index = ~0;
2483   sm->fq_out2in_index = ~0;
2484
2485   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2486   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2487   sm->forwarding_enabled = 0;
2488   sm->log_class = vlib_log_register_class ("nat", 0);
2489   sm->log_level = SNAT_LOG_ERROR;
2490   sm->mss_clamping = 0;
2491
2492   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2493   sm->error_node_index = node->index;
2494
2495   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2496   sm->pre_in2out_node_index = node->index;
2497   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2498   sm->pre_out2in_node_index = node->index;
2499
2500   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2501   sm->pre_in2out_node_index = node->index;
2502
2503   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2504   sm->pre_out2in_node_index = node->index;
2505
2506   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2507   sm->in2out_node_index = node->index;
2508   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2509   sm->in2out_output_node_index = node->index;
2510   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2511   sm->in2out_fast_node_index = node->index;
2512   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2513   sm->in2out_slowpath_node_index = node->index;
2514   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2515   sm->in2out_slowpath_output_node_index = node->index;
2516
2517   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2518   sm->ed_in2out_node_index = node->index;
2519   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2520   sm->ed_in2out_slowpath_node_index = node->index;
2521
2522   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2523   sm->out2in_node_index = node->index;
2524   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2525   sm->out2in_fast_node_index = node->index;
2526
2527   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2528   sm->ed_out2in_node_index = node->index;
2529   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2530   sm->ed_out2in_slowpath_node_index = node->index;
2531
2532   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2533   sm->det_in2out_node_index = node->index;
2534   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2535   sm->det_out2in_node_index = node->index;
2536
2537   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2538   sm->hairpinning_node_index = node->index;
2539   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2540   sm->hairpin_dst_node_index = node->index;
2541   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2542   sm->hairpin_src_node_index = node->index;
2543   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2544   sm->ed_hairpinning_node_index = node->index;
2545   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2546   sm->ed_hairpin_dst_node_index = node->index;
2547   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2548   sm->ed_hairpin_src_node_index = node->index;
2549
2550   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2551   if (p)
2552     {
2553       tr = (vlib_thread_registration_t *) p[0];
2554       if (tr)
2555         {
2556           sm->num_workers = tr->count;
2557           sm->first_worker_index = tr->first_index;
2558         }
2559     }
2560
2561   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2562
2563   /* Use all available workers by default */
2564   if (sm->num_workers > 1)
2565     {
2566       for (i = 0; i < sm->num_workers; i++)
2567         bitmap = clib_bitmap_set (bitmap, i, 1);
2568       snat_set_workers (bitmap);
2569       clib_bitmap_free (bitmap);
2570     }
2571   else
2572     {
2573       sm->per_thread_data[0].snat_thread_index = 0;
2574     }
2575
2576   error = snat_api_init (vm, sm);
2577   if (error)
2578     return error;
2579
2580   /* Set up the interface address add/del callback */
2581   cb4.function = snat_ip4_add_del_interface_address_cb;
2582   cb4.function_opaque = 0;
2583
2584   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2585
2586   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2587   cb4.function_opaque = 0;
2588
2589   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2590
2591   nat_dpo_module_init ();
2592
2593   /* Init counters */
2594   sm->total_users.name = "total-users";
2595   sm->total_users.stat_segment_name = "/nat44/total-users";
2596   vlib_validate_simple_counter (&sm->total_users, 0);
2597   vlib_zero_simple_counter (&sm->total_users, 0);
2598   sm->total_sessions.name = "total-sessions";
2599   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2600   vlib_validate_simple_counter (&sm->total_sessions, 0);
2601   vlib_zero_simple_counter (&sm->total_sessions, 0);
2602
2603   /* Init IPFIX logging */
2604   snat_ipfix_logging_init (vm);
2605
2606   /* Init NAT64 */
2607   error = nat64_init (vm);
2608   if (error)
2609     return error;
2610
2611   nat66_init (vm);
2612
2613   ip4_table_bind_callback_t cbt4 = {
2614     .function = snat_ip4_table_bind,
2615   };
2616   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2617
2618   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2619                                         FIB_SOURCE_PRIORITY_HI,
2620                                         FIB_SOURCE_BH_SIMPLE);
2621   nat_fib_src_low = fib_source_allocate ("nat-low",
2622                                          FIB_SOURCE_PRIORITY_LOW,
2623                                          FIB_SOURCE_BH_SIMPLE);
2624
2625   return error;
2626 }
2627
2628 VLIB_INIT_FUNCTION (snat_init);
2629
2630 void
2631 snat_free_outside_address_and_port (snat_address_t * addresses,
2632                                     u32 thread_index, snat_session_key_t * k)
2633 {
2634   snat_address_t *a;
2635   u32 address_index;
2636   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2637
2638   for (address_index = 0; address_index < vec_len (addresses);
2639        address_index++)
2640     {
2641       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2642         break;
2643     }
2644
2645   ASSERT (address_index < vec_len (addresses));
2646
2647   a = addresses + address_index;
2648
2649   switch (k->protocol)
2650     {
2651 #define _(N, i, n, s) \
2652     case SNAT_PROTOCOL_##N: \
2653       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2654         port_host_byte_order) == 1); \
2655       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2656         port_host_byte_order, 0); \
2657       a->busy_##n##_ports--; \
2658       a->busy_##n##_ports_per_thread[thread_index]--; \
2659       break;
2660       foreach_snat_protocol
2661 #undef _
2662     default:
2663       nat_elog_info ("unknown protocol");
2664       return;
2665     }
2666 }
2667
2668 static int
2669 nat_set_outside_address_and_port (snat_address_t * addresses,
2670                                   u32 thread_index, snat_session_key_t * k)
2671 {
2672   snat_address_t *a = 0;
2673   u32 address_index;
2674   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2675
2676   for (address_index = 0; address_index < vec_len (addresses);
2677        address_index++)
2678     {
2679       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2680         continue;
2681
2682       a = addresses + address_index;
2683       switch (k->protocol)
2684         {
2685 #define _(N, j, n, s) \
2686         case SNAT_PROTOCOL_##N: \
2687           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2688             return VNET_API_ERROR_INSTANCE_IN_USE; \
2689           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2690           a->busy_##n##_ports_per_thread[thread_index]++; \
2691           a->busy_##n##_ports++; \
2692           return 0;
2693           foreach_snat_protocol
2694 #undef _
2695         default:
2696           nat_elog_info ("unknown protocol");
2697           return 1;
2698         }
2699     }
2700
2701   return VNET_API_ERROR_NO_SUCH_ENTRY;
2702 }
2703
2704 int
2705 snat_static_mapping_match (snat_main_t * sm,
2706                            snat_session_key_t match,
2707                            snat_session_key_t * mapping,
2708                            u8 by_external,
2709                            u8 * is_addr_only,
2710                            twice_nat_type_t * twice_nat,
2711                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2712                            u8 * is_identity_nat)
2713 {
2714   clib_bihash_kv_8_8_t kv, value;
2715   snat_static_mapping_t *m;
2716   snat_session_key_t m_key;
2717   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2718   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2719   u8 backend_index;
2720   nat44_lb_addr_port_t *local;
2721
2722   m_key.fib_index = match.fib_index;
2723   if (by_external)
2724     {
2725       mapping_hash = &sm->static_mapping_by_external;
2726       m_key.fib_index = 0;
2727     }
2728
2729   m_key.addr = match.addr;
2730   m_key.port = clib_net_to_host_u16 (match.port);
2731   m_key.protocol = match.protocol;
2732
2733   kv.key = m_key.as_u64;
2734
2735   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2736     {
2737       /* Try address only mapping */
2738       m_key.port = 0;
2739       m_key.protocol = 0;
2740       kv.key = m_key.as_u64;
2741       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2742         return 1;
2743     }
2744
2745   m = pool_elt_at_index (sm->static_mappings, value.value);
2746
2747   if (by_external)
2748     {
2749       if (is_lb_static_mapping (m))
2750         {
2751           if (PREDICT_FALSE (lb != 0))
2752             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2753           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2754                                                           match.addr,
2755                                                           match.protocol,
2756                                                           match.port,
2757                                                           &backend_index))
2758             {
2759               local = pool_elt_at_index (m->locals, backend_index);
2760               mapping->addr = local->addr;
2761               mapping->port = clib_host_to_net_u16 (local->port);
2762               mapping->fib_index = local->fib_index;
2763               goto end;
2764             }
2765           // pick locals matching this worker
2766           if (PREDICT_FALSE (sm->num_workers > 1))
2767             {
2768               u32 thread_index = vlib_get_thread_index ();
2769               /* *INDENT-OFF* */
2770               pool_foreach_index (i, m->locals,
2771               ({
2772                 local = pool_elt_at_index (m->locals, i);
2773
2774                 ip4_header_t ip = {
2775                   .src_address = local->addr,
2776                 };
2777
2778                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2779                     thread_index)
2780                   {
2781                     vec_add1 (tmp, i);
2782                   }
2783               }));
2784               /* *INDENT-ON* */
2785               ASSERT (vec_len (tmp) != 0);
2786             }
2787           else
2788             {
2789               /* *INDENT-OFF* */
2790               pool_foreach_index (i, m->locals,
2791               ({
2792                 vec_add1 (tmp, i);
2793               }));
2794               /* *INDENT-ON* */
2795             }
2796           hi = vec_len (tmp) - 1;
2797           local = pool_elt_at_index (m->locals, tmp[hi]);
2798           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2799           while (lo < hi)
2800             {
2801               mid = ((hi - lo) >> 1) + lo;
2802               local = pool_elt_at_index (m->locals, tmp[mid]);
2803               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2804             }
2805           local = pool_elt_at_index (m->locals, tmp[lo]);
2806           if (!(local->prefix >= rand))
2807             return 1;
2808           mapping->addr = local->addr;
2809           mapping->port = clib_host_to_net_u16 (local->port);
2810           mapping->fib_index = local->fib_index;
2811           if (m->affinity)
2812             {
2813               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2814                                                 match.protocol, match.port,
2815                                                 tmp[lo], m->affinity,
2816                                                 m->affinity_per_service_list_head_index))
2817                 nat_elog_info ("create affinity record failed");
2818             }
2819           vec_free (tmp);
2820         }
2821       else
2822         {
2823           if (PREDICT_FALSE (lb != 0))
2824             *lb = NO_LB_NAT;
2825           mapping->fib_index = m->fib_index;
2826           mapping->addr = m->local_addr;
2827           /* Address only mapping doesn't change port */
2828           mapping->port = is_addr_only_static_mapping (m) ? match.port
2829             : clib_host_to_net_u16 (m->local_port);
2830         }
2831       mapping->protocol = m->proto;
2832     }
2833   else
2834     {
2835       mapping->addr = m->external_addr;
2836       /* Address only mapping doesn't change port */
2837       mapping->port = is_addr_only_static_mapping (m) ? match.port
2838         : clib_host_to_net_u16 (m->external_port);
2839       mapping->fib_index = sm->outside_fib_index;
2840     }
2841
2842 end:
2843   if (PREDICT_FALSE (is_addr_only != 0))
2844     *is_addr_only = is_addr_only_static_mapping (m);
2845
2846   if (PREDICT_FALSE (twice_nat != 0))
2847     *twice_nat = m->twice_nat;
2848
2849   if (PREDICT_FALSE (is_identity_nat != 0))
2850     *is_identity_nat = is_identity_static_mapping (m);
2851
2852   return 0;
2853 }
2854
2855 static_always_inline u16
2856 snat_random_port (u16 min, u16 max)
2857 {
2858   snat_main_t *sm = &snat_main;
2859   return min + random_u32 (&sm->random_seed) /
2860     (random_u32_max () / (max - min + 1) + 1);
2861 }
2862
2863 int
2864 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2865                                      u32 fib_index,
2866                                      u32 thread_index,
2867                                      snat_session_key_t * k,
2868                                      u16 port_per_thread,
2869                                      u32 snat_thread_index)
2870 {
2871   snat_main_t *sm = &snat_main;
2872
2873   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2874                                   port_per_thread, snat_thread_index);
2875 }
2876
2877 static int
2878 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2879                                  u32 fib_index,
2880                                  u32 thread_index,
2881                                  snat_session_key_t * k,
2882                                  u16 port_per_thread, u32 snat_thread_index)
2883 {
2884   int i;
2885   snat_address_t *a, *ga = 0;
2886   u32 portnum;
2887
2888   for (i = 0; i < vec_len (addresses); i++)
2889     {
2890       a = addresses + i;
2891       switch (k->protocol)
2892         {
2893 #define _(N, j, n, s) \
2894         case SNAT_PROTOCOL_##N: \
2895           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2896             { \
2897               if (a->fib_index == fib_index) \
2898                 { \
2899                   while (1) \
2900                     { \
2901                       portnum = (port_per_thread * \
2902                         snat_thread_index) + \
2903                         snat_random_port(1, port_per_thread) + 1024; \
2904                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2905                         continue; \
2906                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2907                       a->busy_##n##_ports_per_thread[thread_index]++; \
2908                       a->busy_##n##_ports++; \
2909                       k->addr = a->addr; \
2910                       k->port = clib_host_to_net_u16(portnum); \
2911                       return 0; \
2912                     } \
2913                 } \
2914               else if (a->fib_index == ~0) \
2915                 { \
2916                   ga = a; \
2917                 } \
2918             } \
2919           break;
2920           foreach_snat_protocol
2921 #undef _
2922         default:
2923           nat_elog_info ("unknown protocol");
2924           return 1;
2925         }
2926
2927     }
2928
2929   if (ga)
2930     {
2931       a = ga;
2932       switch (k->protocol)
2933         {
2934 #define _(N, j, n, s) \
2935         case SNAT_PROTOCOL_##N: \
2936           while (1) \
2937             { \
2938               portnum = (port_per_thread * \
2939                 snat_thread_index) + \
2940                 snat_random_port(1, port_per_thread) + 1024; \
2941               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2942                 continue; \
2943               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2944               a->busy_##n##_ports_per_thread[thread_index]++; \
2945               a->busy_##n##_ports++; \
2946               k->addr = a->addr; \
2947               k->port = clib_host_to_net_u16(portnum); \
2948               return 0; \
2949             }
2950           break;
2951           foreach_snat_protocol
2952 #undef _
2953         default:
2954           nat_elog_info ("unknown protocol");
2955           return 1;
2956         }
2957     }
2958
2959   /* Totally out of translations to use... */
2960   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2961   return 1;
2962 }
2963
2964 static int
2965 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2966                               u32 fib_index,
2967                               u32 thread_index,
2968                               snat_session_key_t * k,
2969                               u16 port_per_thread, u32 snat_thread_index)
2970 {
2971   snat_main_t *sm = &snat_main;
2972   snat_address_t *a = addresses;
2973   u16 m, ports, portnum, A, j;
2974   m = 16 - (sm->psid_offset + sm->psid_length);
2975   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2976
2977   if (!vec_len (addresses))
2978     goto exhausted;
2979
2980   switch (k->protocol)
2981     {
2982 #define _(N, i, n, s) \
2983     case SNAT_PROTOCOL_##N: \
2984       if (a->busy_##n##_ports < ports) \
2985         { \
2986           while (1) \
2987             { \
2988               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2989               j = snat_random_port(0, pow2_mask(m)); \
2990               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2991               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2992                 continue; \
2993               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2994               a->busy_##n##_ports++; \
2995               k->addr = a->addr; \
2996               k->port = clib_host_to_net_u16 (portnum); \
2997               return 0; \
2998             } \
2999         } \
3000       break;
3001       foreach_snat_protocol
3002 #undef _
3003     default:
3004       nat_elog_info ("unknown protocol");
3005       return 1;
3006     }
3007
3008 exhausted:
3009   /* Totally out of translations to use... */
3010   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3011   return 1;
3012 }
3013
3014 static int
3015 nat_alloc_addr_and_port_range (snat_address_t * addresses,
3016                                u32 fib_index,
3017                                u32 thread_index,
3018                                snat_session_key_t * k,
3019                                u16 port_per_thread, u32 snat_thread_index)
3020 {
3021   snat_main_t *sm = &snat_main;
3022   snat_address_t *a = addresses;
3023   u16 portnum, ports;
3024
3025   ports = sm->end_port - sm->start_port + 1;
3026
3027   if (!vec_len (addresses))
3028     goto exhausted;
3029
3030   switch (k->protocol)
3031     {
3032 #define _(N, i, n, s) \
3033     case SNAT_PROTOCOL_##N: \
3034       if (a->busy_##n##_ports < ports) \
3035         { \
3036           while (1) \
3037             { \
3038               portnum = snat_random_port(sm->start_port, sm->end_port); \
3039               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
3040                 continue; \
3041               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
3042               a->busy_##n##_ports++; \
3043               k->addr = a->addr; \
3044               k->port = clib_host_to_net_u16 (portnum); \
3045               return 0; \
3046             } \
3047         } \
3048       break;
3049       foreach_snat_protocol
3050 #undef _
3051     default:
3052       nat_elog_info ("unknown protocol");
3053       return 1;
3054     }
3055
3056 exhausted:
3057   /* Totally out of translations to use... */
3058   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3059   return 1;
3060 }
3061
3062 void
3063 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3064 {
3065   dpo_id_t dpo_v4 = DPO_INVALID;
3066   fib_prefix_t pfx = {
3067     .fp_proto = FIB_PROTOCOL_IP4,
3068     .fp_len = 32,
3069     .fp_addr.ip4.as_u32 = addr.as_u32,
3070   };
3071
3072   if (is_add)
3073     {
3074       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3075       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3076                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3077       dpo_reset (&dpo_v4);
3078     }
3079   else
3080     {
3081       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3082     }
3083 }
3084
3085 u8 *
3086 format_session_kvp (u8 * s, va_list * args)
3087 {
3088   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3089   snat_session_key_t k;
3090
3091   k.as_u64 = v->key;
3092
3093   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
3094
3095   return s;
3096 }
3097
3098 u8 *
3099 format_static_mapping_kvp (u8 * s, va_list * args)
3100 {
3101   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3102   snat_session_key_t k;
3103
3104   k.as_u64 = v->key;
3105
3106   s = format (s, "%U static-mapping-index %llu",
3107               format_static_mapping_key, &k, v->value);
3108
3109   return s;
3110 }
3111
3112 u8 *
3113 format_user_kvp (u8 * s, va_list * args)
3114 {
3115   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3116   snat_user_key_t k;
3117
3118   k.as_u64 = v->key;
3119
3120   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3121               k.fib_index, v->value);
3122
3123   return s;
3124 }
3125
3126 u8 *
3127 format_ed_session_kvp (u8 * s, va_list * args)
3128 {
3129   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3130   nat_ed_ses_key_t k;
3131
3132   k.as_u64[0] = v->key[0];
3133   k.as_u64[1] = v->key[1];
3134
3135   s =
3136     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3137             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
3138             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
3139             format_ip_protocol, k.proto, k.fib_index, v->value);
3140
3141   return s;
3142 }
3143
3144 static u32
3145 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3146                            u8 is_output)
3147 {
3148   snat_main_t *sm = &snat_main;
3149   u32 next_worker_index = 0;
3150   u32 hash;
3151
3152   next_worker_index = sm->first_worker_index;
3153   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3154     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3155
3156   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3157     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3158   else
3159     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3160
3161   return next_worker_index;
3162 }
3163
3164 static u32
3165 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3166                            u32 rx_fib_index0, u8 is_output)
3167 {
3168   snat_main_t *sm = &snat_main;
3169   udp_header_t *udp;
3170   u16 port;
3171   snat_session_key_t m_key;
3172   clib_bihash_kv_8_8_t kv, value;
3173   snat_static_mapping_t *m;
3174   u32 proto;
3175   u32 next_worker_index = 0;
3176
3177   /* first try static mappings without port */
3178   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3179     {
3180       m_key.addr = ip0->dst_address;
3181       m_key.port = 0;
3182       m_key.protocol = 0;
3183       m_key.fib_index = rx_fib_index0;
3184       kv.key = m_key.as_u64;
3185       if (!clib_bihash_search_8_8
3186           (&sm->static_mapping_by_external, &kv, &value))
3187         {
3188           m = pool_elt_at_index (sm->static_mappings, value.value);
3189           return m->workers[0];
3190         }
3191     }
3192
3193   proto = ip_proto_to_snat_proto (ip0->protocol);
3194   udp = ip4_next_header (ip0);
3195   port = udp->dst_port;
3196
3197   /* unknown protocol */
3198   if (PREDICT_FALSE (proto == ~0))
3199     {
3200       /* use current thread */
3201       return vlib_get_thread_index ();
3202     }
3203
3204   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3205     {
3206       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3207       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3208       if (!icmp_type_is_error_message
3209           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3210         port = vnet_buffer (b)->ip.reass.l4_src_port;
3211       else
3212         {
3213           /* if error message, then it's not fragmented and we can access it */
3214           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3215           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3216           void *l4_header = ip4_next_header (inner_ip);
3217           switch (proto)
3218             {
3219             case SNAT_PROTOCOL_ICMP:
3220               icmp = (icmp46_header_t *) l4_header;
3221               echo = (icmp_echo_header_t *) (icmp + 1);
3222               port = echo->identifier;
3223               break;
3224             case SNAT_PROTOCOL_UDP:
3225             case SNAT_PROTOCOL_TCP:
3226               port = ((tcp_udp_header_t *) l4_header)->src_port;
3227               break;
3228             default:
3229               return vlib_get_thread_index ();
3230             }
3231         }
3232     }
3233
3234   /* try static mappings with port */
3235   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3236     {
3237       m_key.addr = ip0->dst_address;
3238       m_key.port = clib_net_to_host_u16 (port);
3239       m_key.protocol = proto;
3240       m_key.fib_index = rx_fib_index0;
3241       kv.key = m_key.as_u64;
3242       if (!clib_bihash_search_8_8
3243           (&sm->static_mapping_by_external, &kv, &value))
3244         {
3245           m = pool_elt_at_index (sm->static_mappings, value.value);
3246           return m->workers[0];
3247         }
3248     }
3249
3250   /* worker by outside port */
3251   next_worker_index = sm->first_worker_index;
3252   next_worker_index +=
3253     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3254   return next_worker_index;
3255 }
3256
3257 static u32
3258 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3259                                u8 is_output)
3260 {
3261   snat_main_t *sm = &snat_main;
3262   u32 next_worker_index = sm->first_worker_index;
3263   u32 hash;
3264
3265   clib_bihash_kv_16_8_t kv16, value16;
3266   snat_main_per_thread_data_t *tsm;
3267   udp_header_t *udp;
3268
3269   if (PREDICT_FALSE (is_output))
3270     {
3271       u32 fib_index = sm->outside_fib_index;
3272       nat_outside_fib_t *outside_fib;
3273       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3274       fib_prefix_t pfx = {
3275         .fp_proto = FIB_PROTOCOL_IP4,
3276         .fp_len = 32,
3277         .fp_addr = {
3278                     .ip4.as_u32 = ip->dst_address.as_u32,
3279                     }
3280         ,
3281       };
3282
3283       udp = ip4_next_header (ip);
3284
3285       switch (vec_len (sm->outside_fibs))
3286         {
3287         case 0:
3288           fib_index = sm->outside_fib_index;
3289           break;
3290         case 1:
3291           fib_index = sm->outside_fibs[0].fib_index;
3292           break;
3293         default:
3294             /* *INDENT-OFF* */
3295             vec_foreach (outside_fib, sm->outside_fibs)
3296               {
3297                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3298                 if (FIB_NODE_INDEX_INVALID != fei)
3299                   {
3300                     if (fib_entry_get_resolving_interface (fei) != ~0)
3301                       {
3302                         fib_index = outside_fib->fib_index;
3303                         break;
3304                       }
3305                   }
3306               }
3307             /* *INDENT-ON* */
3308           break;
3309         }
3310
3311       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3312                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3313
3314       /* *INDENT-OFF* */
3315       vec_foreach (tsm, sm->per_thread_data)
3316         {
3317           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3318                                                       &kv16, &value16)))
3319             {
3320               next_worker_index += tsm->thread_index;
3321
3322               nat_elog_debug_handoff (
3323                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3324                 next_worker_index, fib_index,
3325                 clib_net_to_host_u32 (ip->src_address.as_u32),
3326                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3327
3328               return next_worker_index;
3329             }
3330         }
3331       /* *INDENT-ON* */
3332     }
3333
3334   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3335     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3336
3337   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3338     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3339   else
3340     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3341
3342   if (PREDICT_TRUE (!is_output))
3343     {
3344       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3345                               next_worker_index, rx_fib_index,
3346                               clib_net_to_host_u32 (ip->src_address.as_u32),
3347                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3348     }
3349   else
3350     {
3351       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3352                               next_worker_index, rx_fib_index,
3353                               clib_net_to_host_u32 (ip->src_address.as_u32),
3354                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3355     }
3356
3357   return next_worker_index;
3358 }
3359
3360 static u32
3361 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3362                                u32 rx_fib_index, u8 is_output)
3363 {
3364   snat_main_t *sm = &snat_main;
3365   clib_bihash_kv_8_8_t kv, value;
3366   clib_bihash_kv_16_8_t kv16, value16;
3367   snat_main_per_thread_data_t *tsm;
3368
3369   u32 proto, next_worker_index = 0;
3370   udp_header_t *udp;
3371   u16 port;
3372   snat_static_mapping_t *m;
3373   u32 hash;
3374
3375   proto = ip_proto_to_snat_proto (ip->protocol);
3376
3377   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3378     {
3379       udp = ip4_next_header (ip);
3380
3381       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3382                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3383
3384       /* *INDENT-OFF* */
3385       vec_foreach (tsm, sm->per_thread_data)
3386         {
3387           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3388                                                       &kv16, &value16)))
3389             {
3390               next_worker_index = sm->first_worker_index + tsm->thread_index;
3391               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3392                           next_worker_index, rx_fib_index,
3393                           clib_net_to_host_u32 (ip->src_address.as_u32),
3394                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3395               return next_worker_index;
3396             }
3397           }
3398         /* *INDENT-ON* */
3399     }
3400   else if (proto == SNAT_PROTOCOL_ICMP)
3401     {
3402       nat_ed_ses_key_t key;
3403
3404       if (!get_icmp_o2i_ed_key (b, ip, &key))
3405         {
3406
3407           key.fib_index = rx_fib_index;
3408           kv16.key[0] = key.as_u64[0];
3409           kv16.key[1] = key.as_u64[1];
3410
3411           /* *INDENT-OFF* */
3412           vec_foreach (tsm, sm->per_thread_data)
3413             {
3414               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3415                                                           &kv16, &value16)))
3416                 {
3417                   next_worker_index = sm->first_worker_index +
3418                                       tsm->thread_index;
3419                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3420                               next_worker_index, rx_fib_index,
3421                               clib_net_to_host_u32 (ip->src_address.as_u32),
3422                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3423                   return next_worker_index;
3424                 }
3425             }
3426           /* *INDENT-ON* */
3427         }
3428     }
3429
3430   /* first try static mappings without port */
3431   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3432     {
3433       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3434       if (!clib_bihash_search_8_8
3435           (&sm->static_mapping_by_external, &kv, &value))
3436         {
3437           m = pool_elt_at_index (sm->static_mappings, value.value);
3438           next_worker_index = m->workers[0];
3439           goto done;
3440         }
3441     }
3442
3443   /* unknown protocol */
3444   if (PREDICT_FALSE (proto == ~0))
3445     {
3446       /* use current thread */
3447       next_worker_index = vlib_get_thread_index ();
3448       goto done;
3449     }
3450
3451   udp = ip4_next_header (ip);
3452   port = udp->dst_port;
3453
3454   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3455     {
3456       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3457       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3458       if (!icmp_type_is_error_message
3459           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3460         port = vnet_buffer (b)->ip.reass.l4_src_port;
3461       else
3462         {
3463           /* if error message, then it's not fragmented and we can access it */
3464           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3465           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3466           void *l4_header = ip4_next_header (inner_ip);
3467           switch (proto)
3468             {
3469             case SNAT_PROTOCOL_ICMP:
3470               icmp = (icmp46_header_t *) l4_header;
3471               echo = (icmp_echo_header_t *) (icmp + 1);
3472               port = echo->identifier;
3473               break;
3474             case SNAT_PROTOCOL_UDP:
3475             case SNAT_PROTOCOL_TCP:
3476               port = ((tcp_udp_header_t *) l4_header)->src_port;
3477               break;
3478             default:
3479               next_worker_index = vlib_get_thread_index ();
3480               goto done;
3481             }
3482         }
3483     }
3484
3485   /* try static mappings with port */
3486   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3487     {
3488       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3489                   clib_net_to_host_u16 (port));
3490       if (!clib_bihash_search_8_8
3491           (&sm->static_mapping_by_external, &kv, &value))
3492         {
3493           m = pool_elt_at_index (sm->static_mappings, value.value);
3494           if (!is_lb_static_mapping (m))
3495             {
3496               next_worker_index = m->workers[0];
3497               goto done;
3498             }
3499
3500           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3501             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3502
3503           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3504             next_worker_index =
3505               m->workers[hash & (_vec_len (m->workers) - 1)];
3506           else
3507             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3508           goto done;
3509         }
3510     }
3511
3512   /* worker by outside port */
3513   next_worker_index = sm->first_worker_index;
3514   next_worker_index +=
3515     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3516
3517 done:
3518   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3519                           clib_net_to_host_u32 (ip->src_address.as_u32),
3520                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3521   return next_worker_index;
3522 }
3523
3524 void
3525 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3526                 ip4_address_t * out_addr, u16 out_port,
3527                 ip4_address_t * eh_addr, u16 eh_port,
3528                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3529                 u32 fib_index, u16 flags, u32 thread_index)
3530 {
3531   snat_main_t *sm = &snat_main;
3532   snat_session_key_t key;
3533   snat_user_t *u;
3534   snat_session_t *s;
3535   clib_bihash_kv_8_8_t kv;
3536   f64 now = vlib_time_now (sm->vlib_main);
3537   nat_outside_fib_t *outside_fib;
3538   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3539   snat_main_per_thread_data_t *tsm;
3540   fib_prefix_t pfx = {
3541     .fp_proto = FIB_PROTOCOL_IP4,
3542     .fp_len = 32,
3543     .fp_addr = {
3544                 .ip4.as_u32 = eh_addr->as_u32,
3545                 },
3546   };
3547
3548   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3549
3550   key.addr.as_u32 = out_addr->as_u32;
3551   key.port = out_port;
3552   key.protocol = proto;
3553
3554   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3555     {
3556       if (nat_set_outside_address_and_port
3557           (sm->addresses, thread_index, &key))
3558         return;
3559     }
3560
3561   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3562   if (!u)
3563     return;
3564
3565   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3566   if (!s)
3567     return;
3568
3569   s->last_heard = now;
3570   s->flags = flags;
3571   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3572   s->ext_host_port = eh_port;
3573   user_session_increment (sm, u, snat_is_session_static (s));
3574   switch (vec_len (sm->outside_fibs))
3575     {
3576     case 0:
3577       key.fib_index = sm->outside_fib_index;
3578       break;
3579     case 1:
3580       key.fib_index = sm->outside_fibs[0].fib_index;
3581       break;
3582     default:
3583       /* *INDENT-OFF* */
3584       vec_foreach (outside_fib, sm->outside_fibs)
3585         {
3586           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3587           if (FIB_NODE_INDEX_INVALID != fei)
3588             {
3589               if (fib_entry_get_resolving_interface (fei) != ~0)
3590                 {
3591                   key.fib_index = outside_fib->fib_index;
3592                   break;
3593                 }
3594             }
3595         }
3596       /* *INDENT-ON* */
3597       break;
3598     }
3599   s->out2in = key;
3600   kv.key = key.as_u64;
3601   kv.value = s - tsm->sessions;
3602   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3603     nat_elog_warn ("out2in key add failed");
3604
3605   key.addr.as_u32 = in_addr->as_u32;
3606   key.port = in_port;
3607   key.fib_index = fib_index;
3608   s->in2out = key;
3609   kv.key = key.as_u64;
3610   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3611     nat_elog_warn ("in2out key add failed");
3612 }
3613
3614 void
3615 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3616                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3617                 u32 ti)
3618 {
3619   snat_main_t *sm = &snat_main;
3620   snat_session_key_t key;
3621   clib_bihash_kv_8_8_t kv, value;
3622   u32 thread_index;
3623   snat_session_t *s;
3624   snat_main_per_thread_data_t *tsm;
3625
3626   if (sm->num_workers > 1)
3627     thread_index =
3628       sm->first_worker_index +
3629       (sm->workers[(clib_net_to_host_u16 (out_port) -
3630                     1024) / sm->port_per_thread]);
3631   else
3632     thread_index = sm->num_workers;
3633   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3634
3635   key.addr.as_u32 = out_addr->as_u32;
3636   key.port = out_port;
3637   key.protocol = proto;
3638   key.fib_index = fib_index;
3639   kv.key = key.as_u64;
3640   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3641     return;
3642
3643   s = pool_elt_at_index (tsm->sessions, value.value);
3644   nat_free_session_data (sm, s, thread_index, 1);
3645   nat44_delete_session (sm, s, thread_index);
3646 }
3647
3648 void
3649 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3650                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3651                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3652 {
3653   snat_main_t *sm = &snat_main;
3654   snat_session_key_t key;
3655   clib_bihash_kv_8_8_t kv, value;
3656   snat_session_t *s;
3657   snat_main_per_thread_data_t *tsm;
3658
3659   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3660
3661   key.addr.as_u32 = out_addr->as_u32;
3662   key.port = out_port;
3663   key.protocol = proto;
3664   key.fib_index = fib_index;
3665   kv.key = key.as_u64;
3666   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3667     return;
3668
3669   s = pool_elt_at_index (tsm->sessions, value.value);
3670   s->total_pkts = total_pkts;
3671   s->total_bytes = total_bytes;
3672 }
3673
3674 void
3675 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3676                    ip4_address_t * out_addr, u16 out_port,
3677                    ip4_address_t * eh_addr, u16 eh_port,
3678                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3679                    u32 fib_index, u16 flags, u32 thread_index)
3680 {
3681   snat_main_t *sm = &snat_main;
3682   snat_session_key_t key;
3683   snat_user_t *u;
3684   snat_session_t *s;
3685   clib_bihash_kv_16_8_t kv;
3686   f64 now = vlib_time_now (sm->vlib_main);
3687   nat_outside_fib_t *outside_fib;
3688   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3689   snat_main_per_thread_data_t *tsm;
3690   fib_prefix_t pfx = {
3691     .fp_proto = FIB_PROTOCOL_IP4,
3692     .fp_len = 32,
3693     .fp_addr = {
3694                 .ip4.as_u32 = eh_addr->as_u32,
3695                 },
3696   };
3697
3698   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3699
3700   key.addr.as_u32 = out_addr->as_u32;
3701   key.port = out_port;
3702   key.protocol = proto;
3703
3704   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3705     {
3706       if (nat_set_outside_address_and_port
3707           (sm->addresses, thread_index, &key))
3708         return;
3709     }
3710
3711   key.addr.as_u32 = ehn_addr->as_u32;
3712   key.port = ehn_port;
3713   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3714     {
3715       if (nat_set_outside_address_and_port
3716           (sm->twice_nat_addresses, thread_index, &key))
3717         return;
3718     }
3719
3720   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3721   if (!u)
3722     return;
3723
3724   s = nat_ed_session_alloc (sm, u, thread_index, now);
3725   if (!s)
3726     return;
3727
3728   s->last_heard = now;
3729   s->flags = flags;
3730   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3731   s->ext_host_nat_port = s->ext_host_port = eh_port;
3732   if (is_twice_nat_session (s))
3733     {
3734       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3735       s->ext_host_nat_port = ehn_port;
3736     }
3737   user_session_increment (sm, u, snat_is_session_static (s));
3738   switch (vec_len (sm->outside_fibs))
3739     {
3740     case 0:
3741       key.fib_index = sm->outside_fib_index;
3742       break;
3743     case 1:
3744       key.fib_index = sm->outside_fibs[0].fib_index;
3745       break;
3746     default:
3747       /* *INDENT-OFF* */
3748       vec_foreach (outside_fib, sm->outside_fibs)
3749         {
3750           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3751           if (FIB_NODE_INDEX_INVALID != fei)
3752             {
3753               if (fib_entry_get_resolving_interface (fei) != ~0)
3754                 {
3755                   key.fib_index = outside_fib->fib_index;
3756                   break;
3757                 }
3758             }
3759         }
3760       /* *INDENT-ON* */
3761       break;
3762     }
3763   key.addr.as_u32 = out_addr->as_u32;
3764   key.port = out_port;
3765   s->out2in = key;
3766   kv.value = s - tsm->sessions;
3767
3768   key.addr.as_u32 = in_addr->as_u32;
3769   key.port = in_port;
3770   key.fib_index = fib_index;
3771   s->in2out = key;
3772
3773   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3774               snat_proto_to_ip_proto (proto), fib_index, in_port,
3775               s->ext_host_nat_port);
3776   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3777     nat_elog_warn ("in2out key add failed");
3778
3779   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3780               s->out2in.fib_index, out_port, eh_port);
3781   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3782     nat_elog_warn ("out2in key add failed");
3783 }
3784
3785 void
3786 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3787                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3788                    u32 fib_index, u32 ti)
3789 {
3790   snat_main_t *sm = &snat_main;
3791   nat_ed_ses_key_t key;
3792   clib_bihash_kv_16_8_t kv, value;
3793   u32 thread_index;
3794   snat_session_t *s;
3795   snat_main_per_thread_data_t *tsm;
3796
3797   if (sm->num_workers > 1)
3798     thread_index =
3799       sm->first_worker_index +
3800       (sm->workers[(clib_net_to_host_u16 (out_port) -
3801                     1024) / sm->port_per_thread]);
3802   else
3803     thread_index = sm->num_workers;
3804   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3805
3806   key.l_addr.as_u32 = out_addr->as_u32;
3807   key.l_port = out_port;
3808   key.r_addr.as_u32 = eh_addr->as_u32;
3809   key.r_port = eh_port;
3810   key.proto = proto;
3811   key.fib_index = fib_index;
3812   kv.key[0] = key.as_u64[0];
3813   kv.key[1] = key.as_u64[1];
3814   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3815     return;
3816
3817   s = pool_elt_at_index (tsm->sessions, value.value);
3818   nat_free_session_data (sm, s, thread_index, 1);
3819   nat44_delete_session (sm, s, thread_index);
3820 }
3821
3822 void
3823 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3824                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3825                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3826                    u32 thread_index)
3827 {
3828   snat_main_t *sm = &snat_main;
3829   nat_ed_ses_key_t key;
3830   clib_bihash_kv_16_8_t kv, value;
3831   snat_session_t *s;
3832   snat_main_per_thread_data_t *tsm;
3833
3834   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3835
3836   key.l_addr.as_u32 = out_addr->as_u32;
3837   key.l_port = out_port;
3838   key.r_addr.as_u32 = eh_addr->as_u32;
3839   key.r_port = eh_port;
3840   key.proto = proto;
3841   key.fib_index = fib_index;
3842   kv.key[0] = key.as_u64[0];
3843   kv.key[1] = key.as_u64[1];
3844   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3845     return;
3846
3847   s = pool_elt_at_index (tsm->sessions, value.value);
3848   s->total_pkts = total_pkts;
3849   s->total_bytes = total_bytes;
3850 }
3851
3852 static clib_error_t *
3853 snat_config (vlib_main_t * vm, unformat_input_t * input)
3854 {
3855   snat_main_t *sm = &snat_main;
3856   nat66_main_t *nm = &nat66_main;
3857   //dslite_main_t *dm = &dslite_main;
3858   snat_main_per_thread_data_t *tsm;
3859
3860   u32 static_mapping_buckets = 1024;
3861   u32 static_mapping_memory_size = 64 << 20;
3862
3863   u32 nat64_bib_buckets = 1024;
3864   u32 nat64_bib_memory_size = 128 << 20;
3865
3866   u32 nat64_st_buckets = 2048;
3867   u32 nat64_st_memory_size = 256 << 20;
3868
3869   u32 user_buckets = 128;
3870   u32 user_memory_size = 64 << 20;
3871   u32 translation_buckets = 1024;
3872   u32 translation_memory_size = 128 << 20;
3873
3874   u32 max_translations_per_user = ~0;
3875
3876   u32 outside_vrf_id = 0;
3877   u32 outside_ip6_vrf_id = 0;
3878   u32 inside_vrf_id = 0;
3879   u8 static_mapping_only = 0;
3880   u8 static_mapping_connection_tracking = 0;
3881
3882   // configurable timeouts
3883   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3884   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3885   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3886   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3887
3888   sm->deterministic = 0;
3889   sm->out2in_dpo = 0;
3890   sm->endpoint_dependent = 0;
3891
3892   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3893     {
3894       if (unformat
3895           (input, "translation hash buckets %d", &translation_buckets))
3896         ;
3897       else if (unformat (input, "udp timeout %d", &udp_timeout))
3898         ;
3899       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3900         ;
3901       else if (unformat (input, "tcp transitory timeout %d",
3902                          &tcp_transitory_timeout));
3903       else if (unformat (input, "tcp established timeout %d",
3904                          &tcp_established_timeout));
3905       else if (unformat (input, "translation hash memory %d",
3906                          &translation_memory_size));
3907       else if (unformat (input, "user hash buckets %d", &user_buckets))
3908         ;
3909       else if (unformat (input, "user hash memory %d", &user_memory_size))
3910         ;
3911       else if (unformat (input, "max translations per user %d",
3912                          &max_translations_per_user))
3913         ;
3914       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3915         ;
3916       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3917         ;
3918       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3919         ;
3920       else if (unformat (input, "static mapping only"))
3921         {
3922           static_mapping_only = 1;
3923           if (unformat (input, "connection tracking"))
3924             static_mapping_connection_tracking = 1;
3925         }
3926       else if (unformat (input, "deterministic"))
3927         sm->deterministic = 1;
3928       else if (unformat (input, "nat64 bib hash buckets %d",
3929                          &nat64_bib_buckets))
3930         ;
3931       else if (unformat (input, "nat64 bib hash memory %d",
3932                          &nat64_bib_memory_size))
3933         ;
3934       else
3935         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3936         ;
3937       else if (unformat (input, "nat64 st hash memory %d",
3938                          &nat64_st_memory_size))
3939         ;
3940       else if (unformat (input, "out2in dpo"))
3941         sm->out2in_dpo = 1;
3942       //else if (unformat (input, "dslite ce"))
3943       //dslite_set_ce (dm, 1);
3944       else if (unformat (input, "endpoint-dependent"))
3945         sm->endpoint_dependent = 1;
3946       else
3947         return clib_error_return (0, "unknown input '%U'",
3948                                   format_unformat_error, input);
3949     }
3950
3951   if (sm->deterministic && sm->endpoint_dependent)
3952     return clib_error_return (0,
3953                               "deterministic and endpoint-dependent modes are mutually exclusive");
3954
3955   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3956     return clib_error_return (0,
3957                               "static mapping only mode available only for simple nat");
3958
3959   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3960     return clib_error_return (0,
3961                               "out2in dpo mode available only for simple nat");
3962
3963   /* optionally configurable timeouts for testing purposes */
3964   sm->udp_timeout = udp_timeout;
3965   sm->tcp_transitory_timeout = tcp_transitory_timeout;
3966   sm->tcp_established_timeout = tcp_established_timeout;
3967   sm->icmp_timeout = icmp_timeout;
3968
3969   sm->min_timeout = nat44_minimal_timeout (sm);
3970
3971   sm->user_buckets = user_buckets;
3972   sm->user_memory_size = user_memory_size;
3973
3974   sm->translation_buckets = translation_buckets;
3975   sm->translation_memory_size = translation_memory_size;
3976
3977   /* do not exceed load factor 10 */
3978   sm->max_translations = 10 * translation_buckets;
3979   sm->max_translations_per_user = max_translations_per_user == ~0 ?
3980     sm->max_translations : max_translations_per_user;
3981
3982   sm->outside_vrf_id = outside_vrf_id;
3983   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3984                                                              outside_vrf_id,
3985                                                              nat_fib_src_hi);
3986   nm->outside_vrf_id = outside_ip6_vrf_id;
3987   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3988                                                              outside_ip6_vrf_id,
3989                                                              nat_fib_src_hi);
3990   sm->inside_vrf_id = inside_vrf_id;
3991   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3992                                                             inside_vrf_id,
3993                                                             nat_fib_src_hi);
3994   sm->static_mapping_only = static_mapping_only;
3995   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3996
3997   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3998                   nat64_st_memory_size);
3999
4000   if (sm->deterministic)
4001     {
4002       sm->in2out_node_index = snat_det_in2out_node.index;
4003       sm->in2out_output_node_index = ~0;
4004       sm->out2in_node_index = snat_det_out2in_node.index;
4005       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4006       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4007     }
4008   else
4009     {
4010       if (sm->endpoint_dependent)
4011         {
4012           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4013           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4014
4015           sm->handoff_out2in_index = nat_pre_out2in_node.index;
4016           sm->handoff_in2out_index = nat_pre_in2out_node.index;
4017           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
4018
4019           sm->in2out_node_index = nat44_ed_in2out_node.index;
4020           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4021           sm->out2in_node_index = nat44_ed_out2in_node.index;
4022
4023           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4024           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4025           nat_affinity_init (vm);
4026           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4027                        nat_ha_sref_ed_cb);
4028         }
4029       else
4030         {
4031           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4032           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4033
4034           sm->handoff_out2in_index = snat_out2in_node.index;
4035           sm->handoff_in2out_index = snat_in2out_node.index;
4036           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
4037
4038           sm->in2out_node_index = snat_in2out_node.index;
4039           sm->in2out_output_node_index = snat_in2out_output_node.index;
4040           sm->out2in_node_index = snat_out2in_node.index;
4041           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4042           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4043           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4044         }
4045       if (!static_mapping_only ||
4046           (static_mapping_only && static_mapping_connection_tracking))
4047         {
4048           /* *INDENT-OFF* */
4049           vec_foreach (tsm, sm->per_thread_data)
4050             {
4051               tsm->min_session_timeout = 0;
4052
4053               tsm->cleared = 0;
4054               tsm->cleanup_runs = 0;
4055               tsm->cleanup_timeout = 0;
4056
4057               if (sm->endpoint_dependent)
4058                 {
4059                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
4060                                          translation_buckets,
4061                                          translation_memory_size);
4062                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
4063                                                       format_ed_session_kvp);
4064
4065                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
4066                                          translation_buckets,
4067                                          translation_memory_size);
4068                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
4069                                                       format_ed_session_kvp);
4070                 }
4071               else
4072                 {
4073                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
4074                                         translation_buckets,
4075                                         translation_memory_size);
4076                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
4077                                                      format_session_kvp);
4078
4079                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
4080                                         translation_buckets,
4081                                         translation_memory_size);
4082                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
4083                                                      format_session_kvp);
4084                 }
4085
4086               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
4087                                     user_memory_size);
4088               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
4089                                                  format_user_kvp);
4090             }
4091           /* *INDENT-ON* */
4092
4093         }
4094       else
4095         {
4096           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4097           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4098         }
4099       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4100                             "static_mapping_by_local", static_mapping_buckets,
4101                             static_mapping_memory_size);
4102       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4103                                          format_static_mapping_kvp);
4104
4105       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4106                             "static_mapping_by_external",
4107                             static_mapping_buckets,
4108                             static_mapping_memory_size);
4109       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4110                                          format_static_mapping_kvp);
4111     }
4112
4113   return 0;
4114 }
4115
4116 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4117
4118 static void
4119 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4120                                  uword opaque,
4121                                  u32 sw_if_index,
4122                                  ip4_address_t * address,
4123                                  u32 address_length,
4124                                  u32 if_address_index, u32 is_delete)
4125 {
4126   snat_main_t *sm = &snat_main;
4127   snat_static_map_resolve_t *rp;
4128   snat_static_mapping_t *m;
4129   snat_session_key_t m_key;
4130   clib_bihash_kv_8_8_t kv, value;
4131   int i, rv;
4132   ip4_address_t l_addr;
4133
4134   for (i = 0; i < vec_len (sm->to_resolve); i++)
4135     {
4136       rp = sm->to_resolve + i;
4137       if (rp->addr_only == 0)
4138         continue;
4139       if (rp->sw_if_index == sw_if_index)
4140         goto match;
4141     }
4142
4143   return;
4144
4145 match:
4146   m_key.addr.as_u32 = address->as_u32;
4147   m_key.port = rp->addr_only ? 0 : rp->e_port;
4148   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4149   m_key.fib_index = sm->outside_fib_index;
4150   kv.key = m_key.as_u64;
4151   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4152     m = 0;
4153   else
4154     m = pool_elt_at_index (sm->static_mappings, value.value);
4155
4156   if (!is_delete)
4157     {
4158       /* Don't trip over lease renewal, static config */
4159       if (m)
4160         return;
4161     }
4162   else
4163     {
4164       if (!m)
4165         return;
4166     }
4167
4168   /* Indetity mapping? */
4169   if (rp->l_addr.as_u32 == 0)
4170     l_addr.as_u32 = address[0].as_u32;
4171   else
4172     l_addr.as_u32 = rp->l_addr.as_u32;
4173   /* Add the static mapping */
4174   rv = snat_add_static_mapping (l_addr,
4175                                 address[0],
4176                                 rp->l_port,
4177                                 rp->e_port,
4178                                 rp->vrf_id,
4179                                 rp->addr_only, ~0 /* sw_if_index */ ,
4180                                 rp->proto, !is_delete, rp->twice_nat,
4181                                 rp->out2in_only, rp->tag, rp->identity_nat);
4182   if (rv)
4183     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4184 }
4185
4186 static void
4187 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4188                                        uword opaque,
4189                                        u32 sw_if_index,
4190                                        ip4_address_t * address,
4191                                        u32 address_length,
4192                                        u32 if_address_index, u32 is_delete)
4193 {
4194   snat_main_t *sm = &snat_main;
4195   snat_static_map_resolve_t *rp;
4196   ip4_address_t l_addr;
4197   int i, j;
4198   int rv;
4199   u8 twice_nat = 0;
4200   snat_address_t *addresses = sm->addresses;
4201
4202   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4203     {
4204       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4205         goto match;
4206     }
4207
4208   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4209     {
4210       twice_nat = 1;
4211       addresses = sm->twice_nat_addresses;
4212       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4213         goto match;
4214     }
4215
4216   return;
4217
4218 match:
4219   if (!is_delete)
4220     {
4221       /* Don't trip over lease renewal, static config */
4222       for (j = 0; j < vec_len (addresses); j++)
4223         if (addresses[j].addr.as_u32 == address->as_u32)
4224           return;
4225
4226       (void) snat_add_address (sm, address, ~0, twice_nat);
4227       /* Scan static map resolution vector */
4228       for (j = 0; j < vec_len (sm->to_resolve); j++)
4229         {
4230           rp = sm->to_resolve + j;
4231           if (rp->addr_only)
4232             continue;
4233           /* On this interface? */
4234           if (rp->sw_if_index == sw_if_index)
4235             {
4236               /* Indetity mapping? */
4237               if (rp->l_addr.as_u32 == 0)
4238                 l_addr.as_u32 = address[0].as_u32;
4239               else
4240                 l_addr.as_u32 = rp->l_addr.as_u32;
4241               /* Add the static mapping */
4242               rv = snat_add_static_mapping (l_addr,
4243                                             address[0],
4244                                             rp->l_port,
4245                                             rp->e_port,
4246                                             rp->vrf_id,
4247                                             rp->addr_only,
4248                                             ~0 /* sw_if_index */ ,
4249                                             rp->proto,
4250                                             rp->is_add, rp->twice_nat,
4251                                             rp->out2in_only, rp->tag,
4252                                             rp->identity_nat);
4253               if (rv)
4254                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4255                                     "i4", rv);
4256             }
4257         }
4258       return;
4259     }
4260   else
4261     {
4262       (void) snat_del_address (sm, address[0], 1, twice_nat);
4263       return;
4264     }
4265 }
4266
4267
4268 int
4269 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4270                             u8 twice_nat)
4271 {
4272   ip4_main_t *ip4_main = sm->ip4_main;
4273   ip4_address_t *first_int_addr;
4274   snat_static_map_resolve_t *rp;
4275   u32 *indices_to_delete = 0;
4276   int i, j;
4277   u32 *auto_add_sw_if_indices =
4278     twice_nat ? sm->
4279     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4280
4281   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4282     );
4283
4284   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4285     {
4286       if (auto_add_sw_if_indices[i] == sw_if_index)
4287         {
4288           if (is_del)
4289             {
4290               /* if have address remove it */
4291               if (first_int_addr)
4292                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4293               else
4294                 {
4295                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4296                     {
4297                       rp = sm->to_resolve + j;
4298                       if (rp->sw_if_index == sw_if_index)
4299                         vec_add1 (indices_to_delete, j);
4300                     }
4301                   if (vec_len (indices_to_delete))
4302                     {
4303                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4304                         vec_del1 (sm->to_resolve, j);
4305                       vec_free (indices_to_delete);
4306                     }
4307                 }
4308               if (twice_nat)
4309                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4310               else
4311                 vec_del1 (sm->auto_add_sw_if_indices, i);
4312             }
4313           else
4314             return VNET_API_ERROR_VALUE_EXIST;
4315
4316           return 0;
4317         }
4318     }
4319
4320   if (is_del)
4321     return VNET_API_ERROR_NO_SUCH_ENTRY;
4322
4323   /* add to the auto-address list */
4324   if (twice_nat)
4325     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4326   else
4327     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4328
4329   /* If the address is already bound - or static - add it now */
4330   if (first_int_addr)
4331     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4332
4333   return 0;
4334 }
4335
4336 int
4337 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4338                    snat_protocol_t proto, u32 vrf_id, int is_in)
4339 {
4340   snat_main_per_thread_data_t *tsm;
4341   clib_bihash_kv_8_8_t kv, value;
4342   ip4_header_t ip;
4343   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4344   snat_session_key_t key;
4345   snat_session_t *s;
4346   clib_bihash_8_8_t *t;
4347
4348   if (sm->endpoint_dependent)
4349     return VNET_API_ERROR_UNSUPPORTED;
4350
4351   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4352   if (sm->num_workers > 1)
4353     tsm =
4354       vec_elt_at_index (sm->per_thread_data,
4355                         sm->worker_in2out_cb (&ip, fib_index, 0));
4356   else
4357     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4358
4359   key.addr.as_u32 = addr->as_u32;
4360   key.port = clib_host_to_net_u16 (port);
4361   key.protocol = proto;
4362   key.fib_index = fib_index;
4363   kv.key = key.as_u64;
4364   t = is_in ? &tsm->in2out : &tsm->out2in;
4365   if (!clib_bihash_search_8_8 (t, &kv, &value))
4366     {
4367       if (pool_is_free_index (tsm->sessions, value.value))
4368         return VNET_API_ERROR_UNSPECIFIED;
4369
4370       s = pool_elt_at_index (tsm->sessions, value.value);
4371       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4372       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4373       return 0;
4374     }
4375
4376   return VNET_API_ERROR_NO_SUCH_ENTRY;
4377 }
4378
4379 int
4380 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4381                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4382                       u32 vrf_id, int is_in)
4383 {
4384   ip4_header_t ip;
4385   clib_bihash_16_8_t *t;
4386   nat_ed_ses_key_t key;
4387   clib_bihash_kv_16_8_t kv, value;
4388   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4389   snat_session_t *s;
4390   snat_main_per_thread_data_t *tsm;
4391
4392   if (!sm->endpoint_dependent)
4393     return VNET_API_ERROR_FEATURE_DISABLED;
4394
4395   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4396   if (sm->num_workers > 1)
4397     tsm =
4398       vec_elt_at_index (sm->per_thread_data,
4399                         sm->worker_in2out_cb (&ip, fib_index, 0));
4400   else
4401     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4402
4403   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4404   key.l_addr.as_u32 = addr->as_u32;
4405   key.r_addr.as_u32 = eh_addr->as_u32;
4406   key.l_port = clib_host_to_net_u16 (port);
4407   key.r_port = clib_host_to_net_u16 (eh_port);
4408   key.proto = proto;
4409   key.fib_index = fib_index;
4410   kv.key[0] = key.as_u64[0];
4411   kv.key[1] = key.as_u64[1];
4412   if (clib_bihash_search_16_8 (t, &kv, &value))
4413     return VNET_API_ERROR_NO_SUCH_ENTRY;
4414
4415   if (pool_is_free_index (tsm->sessions, value.value))
4416     return VNET_API_ERROR_UNSPECIFIED;
4417   s = pool_elt_at_index (tsm->sessions, value.value);
4418   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4419   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4420   return 0;
4421 }
4422
4423 void
4424 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4425 {
4426   snat_main_t *sm = &snat_main;
4427
4428   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4429   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4430   sm->psid = psid;
4431   sm->psid_offset = psid_offset;
4432   sm->psid_length = psid_length;
4433 }
4434
4435 void
4436 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4437 {
4438   snat_main_t *sm = &snat_main;
4439
4440   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4441   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4442   sm->start_port = start_port;
4443   sm->end_port = end_port;
4444 }
4445
4446 void
4447 nat_set_alloc_addr_and_port_default (void)
4448 {
4449   snat_main_t *sm = &snat_main;
4450
4451   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4452   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4453 }
4454
4455 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4456                                  vlib_node_runtime_t * node,
4457                                  vlib_frame_t * frame)
4458 {
4459   return 0;
4460 }
4461
4462 /* *INDENT-OFF* */
4463 VLIB_REGISTER_NODE (nat_default_node) = {
4464   .name = "nat-default",
4465   .vector_size = sizeof (u32),
4466   .format_trace = 0,
4467   .type = VLIB_NODE_TYPE_INTERNAL,
4468   .n_errors = 0,
4469   .n_next_nodes = NAT_N_NEXT,
4470   .next_nodes = {
4471     [NAT_NEXT_DROP] = "error-drop",
4472     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4473     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4474     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4475     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4476     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4477     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4478     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4479     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4480   },
4481 };
4482 /* *INDENT-ON* */
4483
4484 /*
4485  * fd.io coding-style-patch-verification: ON
4486  *
4487  * Local Variables:
4488  * eval: (c-set-style "gnu")
4489  * End:
4490  */