src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 fib_source_t nat_fib_src_hi;
  42 fib_source_t nat_fib_src_low;
  43
  44 /* *INDENT-OFF* */
  45 /* Hook up input features */
  46 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat-pre-in2out",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-sv-reassembly-feature"),
  51 };
  52 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat-pre-out2in",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  56                                "ip4-dhcp-client-detect",
  57                                "ip4-sv-reassembly-feature"),
  58 };
  59 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-in2out-worker-handoff",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-out2in-worker-handoff",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-in2out",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  74 };
  75 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-out2in",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  79                                "ip4-dhcp-client-detect"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-classify",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  85 };
  86 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-det-in2out",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-det-out2in",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  95                                "ip4-dhcp-client-detect"),
  96 };
  97 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-det-classify",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-ed-in2out",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 106 };
 107 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-ed-out2in",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 111                                "ip4-dhcp-client-detect"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-ed-classify",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-handoff-classify",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 122 };
 123 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-in2out-fast",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 127 };
 128 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 129   .arc_name = "ip4-unicast",
 130   .node_name = "nat44-out2in-fast",
 131   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 132                                "ip4-dhcp-client-detect"),
 133 };
 134 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 135   .arc_name = "ip4-unicast",
 136   .node_name = "nat44-hairpin-dst",
 137   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 140   .arc_name = "ip4-unicast",
 141   .node_name = "nat44-ed-hairpin-dst",
 142   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 143 };
 144
 145 /* Hook up output features */
 146 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-in2out-output",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-in2out-output-worker-handoff",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 155 };
 156 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 157   .arc_name = "ip4-output",
 158   .node_name = "nat44-hairpin-src",
 159   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 162   .arc_name = "ip4-output",
 163   .node_name = "nat44-ed-in2out-output",
 164   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 165   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 166 };
 167 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 168   .arc_name = "ip4-output",
 169   .node_name = "nat44-ed-hairpin-src",
 170   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 171   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 172 };
 173
 174 /* Hook up ip4-local features */
 175 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 176 {
 177   .arc_name = "ip4-local",
 178   .node_name = "nat44-hairpinning",
 179   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 180 };
 181 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 182 {
 183   .arc_name = "ip4-local",
 184   .node_name = "nat44-ed-hairpinning",
 185   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 186 };
 187
 188
 189 VLIB_PLUGIN_REGISTER () = {
 190     .version = VPP_BUILD_VER,
 191     .description = "Network Address Translation (NAT)",
 192 };
 193 /* *INDENT-ON* */
 194
 195 void
 196 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 197                        u8 is_ha)
 198 {
 199   snat_session_key_t key;
 200   clib_bihash_kv_8_8_t kv;
 201   nat_ed_ses_key_t ed_key;
 202   clib_bihash_kv_16_8_t ed_kv;
 203   snat_main_per_thread_data_t *tsm =
 204     vec_elt_at_index (sm->per_thread_data, thread_index);
 205
 206   if (is_fwd_bypass_session (s))
 207     {
 208       if (snat_is_unk_proto_session (s))
 209         {
 210           ed_key.proto = s->in2out.port;
 211           ed_key.r_port = 0;
 212           ed_key.l_port = 0;
 213         }
 214       else
 215         {
 216           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 217           ed_key.l_port = s->in2out.port;
 218           ed_key.r_port = s->ext_host_port;
 219         }
 220       ed_key.l_addr = s->in2out.addr;
 221       ed_key.r_addr = s->ext_host_addr;
 222       ed_key.fib_index = 0;
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 226         nat_elog_warn ("in2out_ed key del failed");
 227       return;
 228     }
 229
 230   /* session lookup tables */
 231   if (is_ed_session (s))
 232     {
 233       if (is_affinity_sessions (s))
 234         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 235                              s->in2out.protocol, s->out2in.port);
 236       ed_key.l_addr = s->out2in.addr;
 237       ed_key.r_addr = s->ext_host_addr;
 238       ed_key.fib_index = s->out2in.fib_index;
 239       if (snat_is_unk_proto_session (s))
 240         {
 241           ed_key.proto = s->in2out.port;
 242           ed_key.r_port = 0;
 243           ed_key.l_port = 0;
 244         }
 245       else
 246         {
 247           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 248           ed_key.l_port = s->out2in.port;
 249           ed_key.r_port = s->ext_host_port;
 250         }
 251       ed_kv.key[0] = ed_key.as_u64[0];
 252       ed_kv.key[1] = ed_key.as_u64[1];
 253       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 254         nat_elog_warn ("out2in_ed key del failed");
 255       ed_key.l_addr = s->in2out.addr;
 256       ed_key.fib_index = s->in2out.fib_index;
 257       if (!snat_is_unk_proto_session (s))
 258         ed_key.l_port = s->in2out.port;
 259       if (is_twice_nat_session (s))
 260         {
 261           ed_key.r_addr = s->ext_host_nat_addr;
 262           ed_key.r_port = s->ext_host_nat_port;
 263         }
 264       ed_kv.key[0] = ed_key.as_u64[0];
 265       ed_kv.key[1] = ed_key.as_u64[1];
 266       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 267         nat_elog_warn ("in2out_ed key del failed");
 268
 269       if (!is_ha)
 270         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 271                                &s->in2out.addr, s->in2out.port,
 272                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 273                                &s->out2in.addr, s->out2in.port,
 274                                &s->ext_host_addr, s->ext_host_port,
 275                                s->in2out.protocol, is_twice_nat_session (s));
 276     }
 277   else
 278     {
 279       kv.key = s->in2out.as_u64;
 280       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 281         nat_elog_warn ("in2out key del failed");
 282       kv.key = s->out2in.as_u64;
 283       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 284         nat_elog_warn ("out2in key del failed");
 285
 286       if (!is_ha)
 287         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 288                                  &s->in2out.addr, s->in2out.port,
 289                                  &s->out2in.addr, s->out2in.port,
 290                                  s->in2out.protocol);
 291     }
 292
 293   if (snat_is_unk_proto_session (s))
 294     return;
 295
 296   if (!is_ha)
 297     {
 298       /* log NAT event */
 299       snat_ipfix_logging_nat44_ses_delete (thread_index,
 300                                            s->in2out.addr.as_u32,
 301                                            s->out2in.addr.as_u32,
 302                                            s->in2out.protocol,
 303                                            s->in2out.port,
 304                                            s->out2in.port,
 305                                            s->in2out.fib_index);
 306
 307       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 308                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 309                    thread_index);
 310     }
 311
 312   /* Twice NAT address and port for external host */
 313   if (is_twice_nat_session (s))
 314     {
 315       key.protocol = s->in2out.protocol;
 316       key.port = s->ext_host_nat_port;
 317       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 318       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 319                                           thread_index, &key);
 320     }
 321
 322   if (snat_is_session_static (s))
 323     return;
 324
 325   snat_free_outside_address_and_port (sm->addresses, thread_index,
 326                                       &s->out2in);
 327 }
 328
 329 void
 330 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 331                          u32 thread_index, u8 is_ha)
 332 {
 333   snat_session_key_t key;
 334   nat_ed_ses_key_t ed_key;
 335   clib_bihash_kv_16_8_t ed_kv;
 336   snat_main_per_thread_data_t *tsm =
 337     vec_elt_at_index (sm->per_thread_data, thread_index);
 338
 339   if (is_fwd_bypass_session (s))
 340     {
 341       if (snat_is_unk_proto_session (s))
 342         {
 343           ed_key.proto = s->in2out.port;
 344           ed_key.r_port = 0;
 345           ed_key.l_port = 0;
 346         }
 347       else
 348         {
 349           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 350           ed_key.l_port = s->in2out.port;
 351           ed_key.r_port = s->ext_host_port;
 352         }
 353
 354       ed_key.l_addr = s->in2out.addr;
 355       ed_key.r_addr = s->ext_host_addr;
 356       ed_key.fib_index = 0;
 357       ed_kv.key[0] = ed_key.as_u64[0];
 358       ed_kv.key[1] = ed_key.as_u64[1];
 359
 360       if (PREDICT_FALSE
 361           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 362         nat_elog_warn ("in2out_ed key del failed");
 363       return;
 364     }
 365
 366   /* session lookup tables */
 367   if (is_affinity_sessions (s))
 368     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 369                          s->in2out.protocol, s->out2in.port);
 370   ed_key.l_addr = s->out2in.addr;
 371   ed_key.r_addr = s->ext_host_addr;
 372   ed_key.fib_index = s->out2in.fib_index;
 373   if (snat_is_unk_proto_session (s))
 374     {
 375       ed_key.proto = s->in2out.port;
 376       ed_key.r_port = 0;
 377       ed_key.l_port = 0;
 378     }
 379   else
 380     {
 381       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 382       ed_key.l_port = s->out2in.port;
 383       ed_key.r_port = s->ext_host_port;
 384     }
 385   ed_kv.key[0] = ed_key.as_u64[0];
 386   ed_kv.key[1] = ed_key.as_u64[1];
 387
 388   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0)))
 389     nat_elog_warn ("out2in_ed key del failed");
 390
 391   ed_key.l_addr = s->in2out.addr;
 392   ed_key.fib_index = s->in2out.fib_index;
 393
 394   if (!snat_is_unk_proto_session (s))
 395     ed_key.l_port = s->in2out.port;
 396
 397   if (is_twice_nat_session (s))
 398     {
 399       ed_key.r_addr = s->ext_host_nat_addr;
 400       ed_key.r_port = s->ext_host_nat_port;
 401     }
 402
 403   ed_kv.key[0] = ed_key.as_u64[0];
 404   ed_kv.key[1] = ed_key.as_u64[1];
 405
 406   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 407     nat_elog_warn ("in2out_ed key del failed");
 408
 409   if (!is_ha)
 410     {
 411       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 412                              &s->in2out.addr, s->in2out.port,
 413                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 414                              &s->out2in.addr, s->out2in.port,
 415                              &s->ext_host_addr, s->ext_host_port,
 416                              s->in2out.protocol, is_twice_nat_session (s));
 417     }
 418
 419   if (snat_is_unk_proto_session (s))
 420     return;
 421
 422   // is this correct ?
 423   if (!is_ha)
 424     {
 425       snat_ipfix_logging_nat44_ses_delete (thread_index,
 426                                            s->in2out.addr.as_u32,
 427                                            s->out2in.addr.as_u32,
 428                                            s->in2out.protocol,
 429                                            s->in2out.port,
 430                                            s->out2in.port,
 431                                            s->in2out.fib_index);
 432       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 433                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 434                    thread_index);
 435     }
 436
 437   /* Twice NAT address and port for external host */
 438   if (is_twice_nat_session (s))
 439     {
 440       key.protocol = s->in2out.protocol;
 441       key.port = s->ext_host_nat_port;
 442       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 443       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 444                                           thread_index, &key);
 445     }
 446
 447   if (snat_is_session_static (s))
 448     return;
 449
 450   // should be called for every dynamic session
 451   snat_free_outside_address_and_port (sm->addresses, thread_index,
 452                                       &s->out2in);
 453 }
 454
 455
 456 snat_user_t *
 457 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 458                         u32 thread_index)
 459 {
 460   snat_user_t *u = 0;
 461   snat_user_key_t user_key;
 462   clib_bihash_kv_8_8_t kv, value;
 463   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 464   dlist_elt_t *per_user_list_head_elt;
 465
 466   user_key.addr.as_u32 = addr->as_u32;
 467   user_key.fib_index = fib_index;
 468   kv.key = user_key.as_u64;
 469
 470   /* Ever heard of the "user" = src ip4 address before? */
 471   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 472     {
 473       /* no, make a new one */
 474       pool_get (tsm->users, u);
 475       clib_memset (u, 0, sizeof (*u));
 476
 477       u->min_session_timeout = 0;
 478
 479       u->addr.as_u32 = addr->as_u32;
 480       u->fib_index = fib_index;
 481
 482       pool_get (tsm->list_pool, per_user_list_head_elt);
 483
 484       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 485         tsm->list_pool;
 486
 487       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 488
 489       kv.value = u - tsm->users;
 490
 491       /* add user */
 492       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 493         {
 494           nat_elog_warn ("user_hash key add failed");
 495           nat44_delete_user_with_no_session (sm, u, thread_index);
 496           return NULL;
 497         }
 498
 499       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 500                                pool_elts (tsm->users));
 501     }
 502   else
 503     {
 504       u = pool_elt_at_index (tsm->users, value.value);
 505     }
 506
 507   return u;
 508 }
 509
 510 snat_session_t *
 511 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 512                               u32 thread_index, f64 now)
 513 {
 514   snat_session_t *s;
 515   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 516   u32 oldest_per_user_translation_list_index, session_index;
 517   dlist_elt_t *oldest_per_user_translation_list_elt;
 518   dlist_elt_t *per_user_translation_list_elt;
 519
 520   /* Over quota? Recycle the least recently used translation */
 521   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 522     {
 523       oldest_per_user_translation_list_index =
 524         clib_dlist_remove_head (tsm->list_pool,
 525                                 u->sessions_per_user_list_head_index);
 526
 527       ASSERT (oldest_per_user_translation_list_index != ~0);
 528
 529       /* Add it back to the end of the LRU list */
 530       clib_dlist_addtail (tsm->list_pool,
 531                           u->sessions_per_user_list_head_index,
 532                           oldest_per_user_translation_list_index);
 533       /* Get the list element */
 534       oldest_per_user_translation_list_elt =
 535         pool_elt_at_index (tsm->list_pool,
 536                            oldest_per_user_translation_list_index);
 537
 538       /* Get the session index from the list element */
 539       session_index = oldest_per_user_translation_list_elt->value;
 540
 541       /* Get the session */
 542       s = pool_elt_at_index (tsm->sessions, session_index);
 543       nat_free_session_data (sm, s, thread_index, 0);
 544       if (snat_is_session_static (s))
 545         u->nstaticsessions--;
 546       else
 547         u->nsessions--;
 548       s->flags = 0;
 549       s->total_bytes = 0;
 550       s->total_pkts = 0;
 551       s->state = 0;
 552       s->ext_host_addr.as_u32 = 0;
 553       s->ext_host_port = 0;
 554       s->ext_host_nat_addr.as_u32 = 0;
 555       s->ext_host_nat_port = 0;
 556     }
 557   else
 558     {
 559       pool_get (tsm->sessions, s);
 560       clib_memset (s, 0, sizeof (*s));
 561
 562       /* Create list elts */
 563       pool_get (tsm->list_pool, per_user_translation_list_elt);
 564       clib_dlist_init (tsm->list_pool,
 565                        per_user_translation_list_elt - tsm->list_pool);
 566
 567       per_user_translation_list_elt->value = s - tsm->sessions;
 568       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 569       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 570
 571       clib_dlist_addtail (tsm->list_pool,
 572                           s->per_user_list_head_index,
 573                           per_user_translation_list_elt - tsm->list_pool);
 574
 575       s->user_index = u - tsm->users;
 576       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 577                                pool_elts (tsm->sessions));
 578     }
 579
 580   s->ha_last_refreshed = now;
 581
 582   return s;
 583 }
 584
 585 snat_session_t *
 586 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 587                       f64 now)
 588 {
 589   snat_session_t *s;
 590   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 591
 592   dlist_elt_t *oldest_elt;
 593   u64 sess_timeout_time;
 594   u32 oldest_index;
 595
 596   // no sessions
 597   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 598     goto alloc_new;
 599
 600   // no free sessions
 601   if (PREDICT_FALSE
 602       ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user))
 603     {
 604       if (nat44_max_translations_per_user_cleanup (u, thread_index, now))
 605         goto alloc_new;
 606
 607       nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 608                      clib_net_to_host_u32 (u->addr.as_u32));
 609       snat_ipfix_logging_max_entries_per_user (thread_index,
 610                                                sm->max_translations_per_user,
 611                                                u->addr.as_u32);
 612       return 0;
 613     }
 614
 615   oldest_index =
 616     clib_dlist_remove_head (tsm->list_pool,
 617                             u->sessions_per_user_list_head_index);
 618   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 619   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 620
 621   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 622   if (now >= sess_timeout_time ||
 623       (s->tcp_close_timestamp && now >= s->tcp_close_timestamp))
 624     {
 625       // reuse old session
 626       clib_dlist_addtail (tsm->list_pool,
 627                           u->sessions_per_user_list_head_index, oldest_index);
 628       s = nat44_session_reuse_old (sm, u, s, thread_index, now);
 629     }
 630   else
 631     {
 632       // alloc new session
 633       clib_dlist_addhead (tsm->list_pool,
 634                           u->sessions_per_user_list_head_index, oldest_index);
 635     alloc_new:
 636       s = nat44_session_alloc_new (tsm, u, now);
 637       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 638                                pool_elts (tsm->sessions));
 639     }
 640   return s;
 641 }
 642
 643 void
 644 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 645                           int is_add)
 646 {
 647   fib_prefix_t prefix = {
 648     .fp_len = p_len,
 649     .fp_proto = FIB_PROTOCOL_IP4,
 650     .fp_addr = {
 651                 .ip4.as_u32 = addr->as_u32,
 652                 },
 653   };
 654   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 655
 656   if (is_add)
 657     fib_table_entry_update_one_path (fib_index,
 658                                      &prefix,
 659                                      nat_fib_src_low,
 660                                      (FIB_ENTRY_FLAG_CONNECTED |
 661                                       FIB_ENTRY_FLAG_LOCAL |
 662                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 663                                      DPO_PROTO_IP4,
 664                                      NULL,
 665                                      sw_if_index,
 666                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 667   else
 668     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 669 }
 670
 671 int
 672 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 673                   u8 twice_nat)
 674 {
 675   snat_address_t *ap;
 676   snat_interface_t *i;
 677   vlib_thread_main_t *tm = vlib_get_thread_main ();
 678
 679   if (twice_nat && !sm->endpoint_dependent)
 680     return VNET_API_ERROR_FEATURE_DISABLED;
 681
 682   /* Check if address already exists */
 683   /* *INDENT-OFF* */
 684   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 685     {
 686       if (ap->addr.as_u32 == addr->as_u32)
 687         return VNET_API_ERROR_VALUE_EXIST;
 688     }
 689   /* *INDENT-ON* */
 690
 691   if (twice_nat)
 692     vec_add2 (sm->twice_nat_addresses, ap, 1);
 693   else
 694     vec_add2 (sm->addresses, ap, 1);
 695
 696   ap->addr = *addr;
 697   if (vrf_id != ~0)
 698     ap->fib_index =
 699       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 700                                          nat_fib_src_low);
 701   else
 702     ap->fib_index = ~0;
 703 #define _(N, i, n, s) \
 704   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 705   ap->busy_##n##_ports = 0; \
 706   ap->busy_##n##_ports_per_thread = 0;\
 707   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 708   foreach_snat_protocol
 709 #undef _
 710     if (twice_nat)
 711     return 0;
 712
 713   /* Add external address to FIB */
 714   /* *INDENT-OFF* */
 715   pool_foreach (i, sm->interfaces,
 716   ({
 717     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 718       continue;
 719
 720     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 721     break;
 722   }));
 723   pool_foreach (i, sm->output_feature_interfaces,
 724   ({
 725     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 726       continue;
 727
 728     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 729     break;
 730   }));
 731   /* *INDENT-ON* */
 732
 733   return 0;
 734 }
 735
 736 static int
 737 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 738 {
 739   snat_static_mapping_t *m;
 740   /* *INDENT-OFF* */
 741   pool_foreach (m, sm->static_mappings,
 742   ({
 743       if (is_addr_only_static_mapping (m) ||
 744           is_out2in_only_static_mapping (m) ||
 745           is_identity_static_mapping (m))
 746         continue;
 747       if (m->external_addr.as_u32 == addr.as_u32)
 748         return 1;
 749   }));
 750   /* *INDENT-ON* */
 751
 752   return 0;
 753 }
 754
 755 static void
 756 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 757                                        ip4_address_t l_addr,
 758                                        u16 l_port,
 759                                        u32 sw_if_index,
 760                                        u16 e_port,
 761                                        u32 vrf_id,
 762                                        snat_protocol_t proto,
 763                                        int addr_only, int is_add, u8 * tag,
 764                                        int twice_nat, int out2in_only,
 765                                        int identity_nat)
 766 {
 767   snat_static_map_resolve_t *rp;
 768
 769   vec_add2 (sm->to_resolve, rp, 1);
 770   rp->l_addr.as_u32 = l_addr.as_u32;
 771   rp->l_port = l_port;
 772   rp->sw_if_index = sw_if_index;
 773   rp->e_port = e_port;
 774   rp->vrf_id = vrf_id;
 775   rp->proto = proto;
 776   rp->addr_only = addr_only;
 777   rp->is_add = is_add;
 778   rp->twice_nat = twice_nat;
 779   rp->out2in_only = out2in_only;
 780   rp->identity_nat = identity_nat;
 781   rp->tag = vec_dup (tag);
 782 }
 783
 784 static u32
 785 get_thread_idx_by_port (u16 e_port)
 786 {
 787   snat_main_t *sm = &snat_main;
 788   u32 thread_idx = sm->num_workers;
 789   if (sm->num_workers > 1)
 790     {
 791       thread_idx =
 792         sm->first_worker_index +
 793         sm->workers[(e_port - 1024) / sm->port_per_thread];
 794     }
 795   return thread_idx;
 796 }
 797
 798 int
 799 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 800                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 801                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 802                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 803                          u8 identity_nat)
 804 {
 805   snat_main_t *sm = &snat_main;
 806   snat_static_mapping_t *m;
 807   snat_session_key_t m_key;
 808   clib_bihash_kv_8_8_t kv, value;
 809   snat_address_t *a = 0;
 810   u32 fib_index = ~0;
 811   snat_interface_t *interface;
 812   int i;
 813   snat_main_per_thread_data_t *tsm;
 814   snat_user_key_t u_key;
 815   snat_user_t *u;
 816   dlist_elt_t *head, *elt;
 817   u32 elt_index, head_index;
 818   u32 ses_index;
 819   u64 user_index;
 820   snat_session_t *s;
 821   snat_static_map_resolve_t *rp, *rp_match = 0;
 822   nat44_lb_addr_port_t *local;
 823   u32 find = ~0;
 824
 825   if (!sm->endpoint_dependent)
 826     {
 827       if (twice_nat || out2in_only)
 828         return VNET_API_ERROR_FEATURE_DISABLED;
 829     }
 830
 831   /* If the external address is a specific interface address */
 832   if (sw_if_index != ~0)
 833     {
 834       ip4_address_t *first_int_addr;
 835
 836       for (i = 0; i < vec_len (sm->to_resolve); i++)
 837         {
 838           rp = sm->to_resolve + i;
 839           if (rp->sw_if_index != sw_if_index ||
 840               rp->l_addr.as_u32 != l_addr.as_u32 ||
 841               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 842             continue;
 843
 844           if (!addr_only)
 845             {
 846               if ((rp->l_port != l_port && rp->e_port != e_port)
 847                   || rp->proto != proto)
 848                 continue;
 849             }
 850
 851           rp_match = rp;
 852           break;
 853         }
 854
 855       /* Might be already set... */
 856       first_int_addr = ip4_interface_first_address
 857         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 858
 859       if (is_add)
 860         {
 861           if (rp_match)
 862             return VNET_API_ERROR_VALUE_EXIST;
 863
 864           snat_add_static_mapping_when_resolved
 865             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 866              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 867
 868           /* DHCP resolution required? */
 869           if (first_int_addr == 0)
 870             {
 871               return 0;
 872             }
 873           else
 874             {
 875               e_addr.as_u32 = first_int_addr->as_u32;
 876               /* Identity mapping? */
 877               if (l_addr.as_u32 == 0)
 878                 l_addr.as_u32 = e_addr.as_u32;
 879             }
 880         }
 881       else
 882         {
 883           if (!rp_match)
 884             return VNET_API_ERROR_NO_SUCH_ENTRY;
 885
 886           vec_del1 (sm->to_resolve, i);
 887
 888           if (first_int_addr)
 889             {
 890               e_addr.as_u32 = first_int_addr->as_u32;
 891               /* Identity mapping? */
 892               if (l_addr.as_u32 == 0)
 893                 l_addr.as_u32 = e_addr.as_u32;
 894             }
 895           else
 896             return 0;
 897         }
 898     }
 899
 900   m_key.addr = e_addr;
 901   m_key.port = addr_only ? 0 : e_port;
 902   m_key.protocol = addr_only ? 0 : proto;
 903   m_key.fib_index = 0;
 904   kv.key = m_key.as_u64;
 905   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 906     m = 0;
 907   else
 908     m = pool_elt_at_index (sm->static_mappings, value.value);
 909
 910   if (is_add)
 911     {
 912       if (m)
 913         {
 914           if (is_identity_static_mapping (m))
 915             {
 916               /* *INDENT-OFF* */
 917               pool_foreach (local, m->locals,
 918               ({
 919                 if (local->vrf_id == vrf_id)
 920                   return VNET_API_ERROR_VALUE_EXIST;
 921               }));
 922               /* *INDENT-ON* */
 923               pool_get (m->locals, local);
 924               local->vrf_id = vrf_id;
 925               local->fib_index =
 926                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 927                                                    nat_fib_src_low);
 928               m_key.addr = m->local_addr;
 929               m_key.port = m->local_port;
 930               m_key.protocol = m->proto;
 931               m_key.fib_index = local->fib_index;
 932               kv.key = m_key.as_u64;
 933               kv.value = m - sm->static_mappings;
 934               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 935               return 0;
 936             }
 937           else
 938             return VNET_API_ERROR_VALUE_EXIST;
 939         }
 940
 941       if (twice_nat && addr_only)
 942         return VNET_API_ERROR_UNSUPPORTED;
 943
 944       /* Convert VRF id to FIB index */
 945       if (vrf_id != ~0)
 946         fib_index =
 947           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 948                                              nat_fib_src_low);
 949       /* If not specified use inside VRF id from SNAT plugin startup config */
 950       else
 951         {
 952           fib_index = sm->inside_fib_index;
 953           vrf_id = sm->inside_vrf_id;
 954           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 955         }
 956
 957       if (!(out2in_only || identity_nat))
 958         {
 959           m_key.addr = l_addr;
 960           m_key.port = addr_only ? 0 : l_port;
 961           m_key.protocol = addr_only ? 0 : proto;
 962           m_key.fib_index = fib_index;
 963           kv.key = m_key.as_u64;
 964           if (!clib_bihash_search_8_8
 965               (&sm->static_mapping_by_local, &kv, &value))
 966             return VNET_API_ERROR_VALUE_EXIST;
 967         }
 968
 969       /* Find external address in allocated addresses and reserve port for
 970          address and port pair mapping when dynamic translations enabled */
 971       if (!(addr_only || sm->static_mapping_only || out2in_only))
 972         {
 973           for (i = 0; i < vec_len (sm->addresses); i++)
 974             {
 975               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 976                 {
 977                   a = sm->addresses + i;
 978                   /* External port must be unused */
 979                   switch (proto)
 980                     {
 981 #define _(N, j, n, s) \
 982                     case SNAT_PROTOCOL_##N: \
 983                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 984                         return VNET_API_ERROR_INVALID_VALUE; \
 985                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 986                       if (e_port > 1024) \
 987                         { \
 988                           a->busy_##n##_ports++; \
 989                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 990                         } \
 991                       break;
 992                       foreach_snat_protocol
 993 #undef _
 994                     default:
 995                       nat_elog_info ("unknown protocol");
 996                       return VNET_API_ERROR_INVALID_VALUE_2;
 997                     }
 998                   break;
 999                 }
1000             }
1001           /* External address must be allocated */
1002           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1003             {
1004               if (sw_if_index != ~0)
1005                 {
1006                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1007                     {
1008                       rp = sm->to_resolve + i;
1009                       if (rp->addr_only)
1010                         continue;
1011                       if (rp->sw_if_index != sw_if_index &&
1012                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1013                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1014                           rp->e_port != e_port && rp->proto != proto)
1015                         continue;
1016
1017                       vec_del1 (sm->to_resolve, i);
1018                       break;
1019                     }
1020                 }
1021               return VNET_API_ERROR_NO_SUCH_ENTRY;
1022             }
1023         }
1024
1025       pool_get (sm->static_mappings, m);
1026       clib_memset (m, 0, sizeof (*m));
1027       m->tag = vec_dup (tag);
1028       m->local_addr = l_addr;
1029       m->external_addr = e_addr;
1030       m->twice_nat = twice_nat;
1031       if (out2in_only)
1032         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1033       if (addr_only)
1034         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1035       if (identity_nat)
1036         {
1037           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1038           pool_get (m->locals, local);
1039           local->vrf_id = vrf_id;
1040           local->fib_index = fib_index;
1041         }
1042       else
1043         {
1044           m->vrf_id = vrf_id;
1045           m->fib_index = fib_index;
1046         }
1047       if (!addr_only)
1048         {
1049           m->local_port = l_port;
1050           m->external_port = e_port;
1051           m->proto = proto;
1052         }
1053
1054       if (sm->num_workers > 1)
1055         {
1056           ip4_header_t ip = {
1057             .src_address = m->local_addr,
1058           };
1059           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1060           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1061         }
1062       else
1063         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1064
1065       m_key.addr = m->local_addr;
1066       m_key.port = m->local_port;
1067       m_key.protocol = m->proto;
1068       m_key.fib_index = fib_index;
1069       kv.key = m_key.as_u64;
1070       kv.value = m - sm->static_mappings;
1071       if (!out2in_only)
1072         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1073
1074       m_key.addr = m->external_addr;
1075       m_key.port = m->external_port;
1076       m_key.fib_index = 0;
1077       kv.key = m_key.as_u64;
1078       kv.value = m - sm->static_mappings;
1079       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1080
1081       /* Delete dynamic sessions matching local address (+ local port) */
1082       if (!(sm->static_mapping_only))
1083         {
1084           u_key.addr = m->local_addr;
1085           u_key.fib_index = m->fib_index;
1086           kv.key = u_key.as_u64;
1087           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1088             {
1089               user_index = value.value;
1090               u = pool_elt_at_index (tsm->users, user_index);
1091               if (u->nsessions)
1092                 {
1093                   head_index = u->sessions_per_user_list_head_index;
1094                   head = pool_elt_at_index (tsm->list_pool, head_index);
1095                   elt_index = head->next;
1096                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1097                   ses_index = elt->value;
1098                   while (ses_index != ~0)
1099                     {
1100                       s = pool_elt_at_index (tsm->sessions, ses_index);
1101                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1102                       ses_index = elt->value;
1103
1104                       if (snat_is_session_static (s))
1105                         continue;
1106
1107                       if (!addr_only
1108                           && (clib_net_to_host_u16 (s->in2out.port) !=
1109                               m->local_port))
1110                         continue;
1111
1112                       nat_free_session_data (sm, s,
1113                                              tsm - sm->per_thread_data, 0);
1114                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1115
1116                       if (!addr_only && !sm->endpoint_dependent)
1117                         break;
1118                     }
1119                 }
1120             }
1121         }
1122     }
1123   else
1124     {
1125       if (!m)
1126         {
1127           if (sw_if_index != ~0)
1128             return 0;
1129           else
1130             return VNET_API_ERROR_NO_SUCH_ENTRY;
1131         }
1132
1133       if (identity_nat)
1134         {
1135           if (vrf_id == ~0)
1136             vrf_id = sm->inside_vrf_id;
1137
1138           /* *INDENT-OFF* */
1139           pool_foreach (local, m->locals,
1140           ({
1141             if (local->vrf_id == vrf_id)
1142               find = local - m->locals;
1143           }));
1144           /* *INDENT-ON* */
1145           if (find == ~0)
1146             return VNET_API_ERROR_NO_SUCH_ENTRY;
1147
1148           local = pool_elt_at_index (m->locals, find);
1149           fib_index = local->fib_index;
1150           pool_put (m->locals, local);
1151         }
1152       else
1153         fib_index = m->fib_index;
1154
1155       /* Free external address port */
1156       if (!(addr_only || sm->static_mapping_only || out2in_only))
1157         {
1158           for (i = 0; i < vec_len (sm->addresses); i++)
1159             {
1160               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1161                 {
1162                   a = sm->addresses + i;
1163                   switch (proto)
1164                     {
1165 #define _(N, j, n, s) \
1166                     case SNAT_PROTOCOL_##N: \
1167                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1168                       if (e_port > 1024) \
1169                         { \
1170                           a->busy_##n##_ports--; \
1171                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1172                         } \
1173                       break;
1174                       foreach_snat_protocol
1175 #undef _
1176                     default:
1177                       nat_elog_info ("unknown protocol");
1178                       return VNET_API_ERROR_INVALID_VALUE_2;
1179                     }
1180                   break;
1181                 }
1182             }
1183         }
1184
1185       if (sm->num_workers > 1)
1186         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1187       else
1188         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1189
1190       m_key.addr = m->local_addr;
1191       m_key.port = m->local_port;
1192       m_key.protocol = m->proto;
1193       m_key.fib_index = fib_index;
1194       kv.key = m_key.as_u64;
1195       if (!out2in_only)
1196         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1197
1198       /* Delete session(s) for static mapping if exist */
1199       if (!(sm->static_mapping_only) ||
1200           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1201         {
1202           u_key.addr = m->local_addr;
1203           u_key.fib_index = fib_index;
1204           kv.key = u_key.as_u64;
1205           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1206             {
1207               user_index = value.value;
1208               u = pool_elt_at_index (tsm->users, user_index);
1209               if (u->nstaticsessions)
1210                 {
1211                   head_index = u->sessions_per_user_list_head_index;
1212                   head = pool_elt_at_index (tsm->list_pool, head_index);
1213                   elt_index = head->next;
1214                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1215                   ses_index = elt->value;
1216                   while (ses_index != ~0)
1217                     {
1218                       s = pool_elt_at_index (tsm->sessions, ses_index);
1219                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1220                       ses_index = elt->value;
1221
1222                       if (!addr_only)
1223                         {
1224                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1225                               (clib_net_to_host_u16 (s->out2in.port) !=
1226                                e_port))
1227                             continue;
1228                         }
1229
1230                       if (is_lb_session (s))
1231                         continue;
1232
1233                       if (!snat_is_session_static (s))
1234                         continue;
1235
1236                       nat_free_session_data (sm, s,
1237                                              tsm - sm->per_thread_data, 0);
1238                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1239
1240                       if (!addr_only && !sm->endpoint_dependent)
1241                         break;
1242                     }
1243                 }
1244             }
1245         }
1246
1247       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1248       if (pool_elts (m->locals))
1249         return 0;
1250
1251       m_key.addr = m->external_addr;
1252       m_key.port = m->external_port;
1253       m_key.fib_index = 0;
1254       kv.key = m_key.as_u64;
1255       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1256
1257       vec_free (m->tag);
1258       vec_free (m->workers);
1259       /* Delete static mapping from pool */
1260       pool_put (sm->static_mappings, m);
1261     }
1262
1263   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1264     return 0;
1265
1266   /* Add/delete external address to FIB */
1267   /* *INDENT-OFF* */
1268   pool_foreach (interface, sm->interfaces,
1269   ({
1270     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1271       continue;
1272
1273     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1274     break;
1275   }));
1276   pool_foreach (interface, sm->output_feature_interfaces,
1277   ({
1278     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1279       continue;
1280
1281     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1282     break;
1283   }));
1284   /* *INDENT-ON* */
1285
1286   return 0;
1287 }
1288
1289 int
1290 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1291                                  snat_protocol_t proto,
1292                                  nat44_lb_addr_port_t * locals, u8 is_add,
1293                                  twice_nat_type_t twice_nat, u8 out2in_only,
1294                                  u8 * tag, u32 affinity)
1295 {
1296   snat_main_t *sm = &snat_main;
1297   snat_static_mapping_t *m;
1298   snat_session_key_t m_key;
1299   clib_bihash_kv_8_8_t kv, value;
1300   snat_address_t *a = 0;
1301   int i;
1302   nat44_lb_addr_port_t *local;
1303   u32 elt_index, head_index, ses_index;
1304   snat_main_per_thread_data_t *tsm;
1305   snat_user_key_t u_key;
1306   snat_user_t *u;
1307   snat_session_t *s;
1308   dlist_elt_t *head, *elt;
1309   uword *bitmap = 0;
1310
1311   if (!sm->endpoint_dependent)
1312     return VNET_API_ERROR_FEATURE_DISABLED;
1313
1314   m_key.addr = e_addr;
1315   m_key.port = e_port;
1316   m_key.protocol = proto;
1317   m_key.fib_index = 0;
1318   kv.key = m_key.as_u64;
1319   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1320     m = 0;
1321   else
1322     m = pool_elt_at_index (sm->static_mappings, value.value);
1323
1324   if (is_add)
1325     {
1326       if (m)
1327         return VNET_API_ERROR_VALUE_EXIST;
1328
1329       if (vec_len (locals) < 2)
1330         return VNET_API_ERROR_INVALID_VALUE;
1331
1332       /* Find external address in allocated addresses and reserve port for
1333          address and port pair mapping when dynamic translations enabled */
1334       if (!(sm->static_mapping_only || out2in_only))
1335         {
1336           for (i = 0; i < vec_len (sm->addresses); i++)
1337             {
1338               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1339                 {
1340                   a = sm->addresses + i;
1341                   /* External port must be unused */
1342                   switch (proto)
1343                     {
1344 #define _(N, j, n, s) \
1345                     case SNAT_PROTOCOL_##N: \
1346                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1347                         return VNET_API_ERROR_INVALID_VALUE; \
1348                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1349                       if (e_port > 1024) \
1350                         { \
1351                           a->busy_##n##_ports++; \
1352                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1353                         } \
1354                       break;
1355                       foreach_snat_protocol
1356 #undef _
1357                     default:
1358                       nat_elog_info ("unknown protocol");
1359                       return VNET_API_ERROR_INVALID_VALUE_2;
1360                     }
1361                   break;
1362                 }
1363             }
1364           /* External address must be allocated */
1365           if (!a)
1366             return VNET_API_ERROR_NO_SUCH_ENTRY;
1367         }
1368
1369       pool_get (sm->static_mappings, m);
1370       clib_memset (m, 0, sizeof (*m));
1371       m->tag = vec_dup (tag);
1372       m->external_addr = e_addr;
1373       m->external_port = e_port;
1374       m->proto = proto;
1375       m->twice_nat = twice_nat;
1376       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1377       if (out2in_only)
1378         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1379       m->affinity = affinity;
1380
1381       if (affinity)
1382         m->affinity_per_service_list_head_index =
1383           nat_affinity_get_per_service_list_head_index ();
1384       else
1385         m->affinity_per_service_list_head_index = ~0;
1386
1387       m_key.addr = m->external_addr;
1388       m_key.port = m->external_port;
1389       m_key.protocol = m->proto;
1390       m_key.fib_index = 0;
1391       kv.key = m_key.as_u64;
1392       kv.value = m - sm->static_mappings;
1393       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1394         {
1395           nat_elog_err ("static_mapping_by_external key add failed");
1396           return VNET_API_ERROR_UNSPECIFIED;
1397         }
1398
1399       m_key.fib_index = m->fib_index;
1400       for (i = 0; i < vec_len (locals); i++)
1401         {
1402           locals[i].fib_index =
1403             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1404                                                locals[i].vrf_id,
1405                                                nat_fib_src_low);
1406           m_key.addr = locals[i].addr;
1407           m_key.fib_index = locals[i].fib_index;
1408           if (!out2in_only)
1409             {
1410               m_key.port = locals[i].port;
1411               kv.key = m_key.as_u64;
1412               kv.value = m - sm->static_mappings;
1413               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1414             }
1415           locals[i].prefix = (i == 0) ? locals[i].probability :
1416             (locals[i - 1].prefix + locals[i].probability);
1417           pool_get (m->locals, local);
1418           *local = locals[i];
1419           if (sm->num_workers > 1)
1420             {
1421               ip4_header_t ip = {
1422                 .src_address = locals[i].addr,
1423               };
1424               bitmap =
1425                 clib_bitmap_set (bitmap,
1426                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1427                                  1);
1428             }
1429         }
1430
1431       /* Assign workers */
1432       if (sm->num_workers > 1)
1433         {
1434           /* *INDENT-OFF* */
1435           clib_bitmap_foreach (i, bitmap,
1436             ({
1437                vec_add1(m->workers, i);
1438             }));
1439           /* *INDENT-ON* */
1440         }
1441     }
1442   else
1443     {
1444       if (!m)
1445         return VNET_API_ERROR_NO_SUCH_ENTRY;
1446
1447       if (!is_lb_static_mapping (m))
1448         return VNET_API_ERROR_INVALID_VALUE;
1449
1450       /* Free external address port */
1451       if (!(sm->static_mapping_only || out2in_only))
1452         {
1453           for (i = 0; i < vec_len (sm->addresses); i++)
1454             {
1455               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1456                 {
1457                   a = sm->addresses + i;
1458                   switch (proto)
1459                     {
1460 #define _(N, j, n, s) \
1461                     case SNAT_PROTOCOL_##N: \
1462                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1463                       if (e_port > 1024) \
1464                         { \
1465                           a->busy_##n##_ports--; \
1466                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1467                         } \
1468                       break;
1469                       foreach_snat_protocol
1470 #undef _
1471                     default:
1472                       nat_elog_info ("unknown protocol");
1473                       return VNET_API_ERROR_INVALID_VALUE_2;
1474                     }
1475                   break;
1476                 }
1477             }
1478         }
1479
1480       m_key.addr = m->external_addr;
1481       m_key.port = m->external_port;
1482       m_key.protocol = m->proto;
1483       m_key.fib_index = 0;
1484       kv.key = m_key.as_u64;
1485       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1486         {
1487           nat_elog_err ("static_mapping_by_external key del failed");
1488           return VNET_API_ERROR_UNSPECIFIED;
1489         }
1490
1491       /* *INDENT-OFF* */
1492       pool_foreach (local, m->locals,
1493       ({
1494           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1495                             nat_fib_src_low);
1496           m_key.addr = local->addr;
1497           if (!out2in_only)
1498             {
1499               m_key.port = local->port;
1500               m_key.fib_index = local->fib_index;
1501               kv.key = m_key.as_u64;
1502               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1503                 {
1504                   nat_elog_err ("static_mapping_by_local key del failed");
1505                   return VNET_API_ERROR_UNSPECIFIED;
1506                 }
1507             }
1508
1509           if (sm->num_workers > 1)
1510             {
1511               ip4_header_t ip = {
1512                 .src_address = local->addr,
1513               };
1514               tsm = vec_elt_at_index (sm->per_thread_data,
1515                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1516             }
1517           else
1518             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1519
1520           /* Delete sessions */
1521           u_key.addr = local->addr;
1522           u_key.fib_index = local->fib_index;
1523           kv.key = u_key.as_u64;
1524           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1525             {
1526               u = pool_elt_at_index (tsm->users, value.value);
1527               if (u->nstaticsessions)
1528                 {
1529                   head_index = u->sessions_per_user_list_head_index;
1530                   head = pool_elt_at_index (tsm->list_pool, head_index);
1531                   elt_index = head->next;
1532                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1533                   ses_index = elt->value;
1534                   while (ses_index != ~0)
1535                     {
1536                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1537                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1538                       ses_index = elt->value;
1539
1540                       if (!(is_lb_session (s)))
1541                         continue;
1542
1543                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1544                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1545                         continue;
1546
1547                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1548                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1549                     }
1550                 }
1551             }
1552       }));
1553       /* *INDENT-ON* */
1554       if (m->affinity)
1555         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1556       pool_free (m->locals);
1557       vec_free (m->tag);
1558       vec_free (m->workers);
1559
1560       pool_put (sm->static_mappings, m);
1561     }
1562
1563   return 0;
1564 }
1565
1566 int
1567 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1568                                        ip4_address_t l_addr, u16 l_port,
1569                                        snat_protocol_t proto, u32 vrf_id,
1570                                        u8 probability, u8 is_add)
1571 {
1572   snat_main_t *sm = &snat_main;
1573   snat_static_mapping_t *m = 0;
1574   snat_session_key_t m_key;
1575   clib_bihash_kv_8_8_t kv, value;
1576   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1577   snat_main_per_thread_data_t *tsm;
1578   snat_user_key_t u_key;
1579   snat_user_t *u;
1580   snat_session_t *s;
1581   dlist_elt_t *head, *elt;
1582   u32 elt_index, head_index, ses_index, *locals = 0;
1583   uword *bitmap = 0;
1584   int i;
1585
1586   if (!sm->endpoint_dependent)
1587     return VNET_API_ERROR_FEATURE_DISABLED;
1588
1589   m_key.addr = e_addr;
1590   m_key.port = e_port;
1591   m_key.protocol = proto;
1592   m_key.fib_index = 0;
1593   kv.key = m_key.as_u64;
1594   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1595     m = pool_elt_at_index (sm->static_mappings, value.value);
1596
1597   if (!m)
1598     return VNET_API_ERROR_NO_SUCH_ENTRY;
1599
1600   if (!is_lb_static_mapping (m))
1601     return VNET_API_ERROR_INVALID_VALUE;
1602
1603   /* *INDENT-OFF* */
1604   pool_foreach (local, m->locals,
1605   ({
1606     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1607         (local->vrf_id == vrf_id))
1608       {
1609         match_local = local;
1610         break;
1611       }
1612   }));
1613   /* *INDENT-ON* */
1614
1615   if (is_add)
1616     {
1617       if (match_local)
1618         return VNET_API_ERROR_VALUE_EXIST;
1619
1620       pool_get (m->locals, local);
1621       clib_memset (local, 0, sizeof (*local));
1622       local->addr.as_u32 = l_addr.as_u32;
1623       local->port = l_port;
1624       local->probability = probability;
1625       local->vrf_id = vrf_id;
1626       local->fib_index =
1627         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1628                                            nat_fib_src_low);
1629
1630       if (!is_out2in_only_static_mapping (m))
1631         {
1632           m_key.addr = l_addr;
1633           m_key.port = l_port;
1634           m_key.fib_index = local->fib_index;
1635           kv.key = m_key.as_u64;
1636           kv.value = m - sm->static_mappings;
1637           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1638             nat_elog_err ("static_mapping_by_local key add failed");
1639         }
1640     }
1641   else
1642     {
1643       if (!match_local)
1644         return VNET_API_ERROR_NO_SUCH_ENTRY;
1645
1646       if (pool_elts (m->locals) < 3)
1647         return VNET_API_ERROR_UNSPECIFIED;
1648
1649       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1650                         nat_fib_src_low);
1651
1652       if (!is_out2in_only_static_mapping (m))
1653         {
1654           m_key.addr = l_addr;
1655           m_key.port = l_port;
1656           m_key.fib_index = match_local->fib_index;
1657           kv.key = m_key.as_u64;
1658           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1659             nat_elog_err ("static_mapping_by_local key del failed");
1660         }
1661
1662       if (sm->num_workers > 1)
1663         {
1664           ip4_header_t ip = {
1665             .src_address = local->addr,
1666           };
1667           tsm = vec_elt_at_index (sm->per_thread_data,
1668                                   sm->worker_in2out_cb (&ip, m->fib_index,
1669                                                         0));
1670         }
1671       else
1672         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1673
1674       /* Delete sessions */
1675       u_key.addr = match_local->addr;
1676       u_key.fib_index = match_local->fib_index;
1677       kv.key = u_key.as_u64;
1678       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1679         {
1680           u = pool_elt_at_index (tsm->users, value.value);
1681           if (u->nstaticsessions)
1682             {
1683               head_index = u->sessions_per_user_list_head_index;
1684               head = pool_elt_at_index (tsm->list_pool, head_index);
1685               elt_index = head->next;
1686               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1687               ses_index = elt->value;
1688               while (ses_index != ~0)
1689                 {
1690                   s = pool_elt_at_index (tsm->sessions, ses_index);
1691                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1692                   ses_index = elt->value;
1693
1694                   if (!(is_lb_session (s)))
1695                     continue;
1696
1697                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1698                       (clib_net_to_host_u16 (s->in2out.port) !=
1699                        match_local->port))
1700                     continue;
1701
1702                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1703                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1704                 }
1705             }
1706         }
1707
1708       pool_put (m->locals, match_local);
1709     }
1710
1711   vec_free (m->workers);
1712
1713   /* *INDENT-OFF* */
1714   pool_foreach (local, m->locals,
1715   ({
1716     vec_add1 (locals, local - m->locals);
1717     if (sm->num_workers > 1)
1718       {
1719         ip4_header_t ip;
1720         ip.src_address.as_u32 = local->addr.as_u32,
1721         bitmap = clib_bitmap_set (bitmap,
1722                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1723                                   1);
1724       }
1725   }));
1726   /* *INDENT-ON* */
1727
1728   ASSERT (vec_len (locals) > 1);
1729
1730   local = pool_elt_at_index (m->locals, locals[0]);
1731   local->prefix = local->probability;
1732   for (i = 1; i < vec_len (locals); i++)
1733     {
1734       local = pool_elt_at_index (m->locals, locals[i]);
1735       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1736       local->prefix = local->probability + prev_local->prefix;
1737     }
1738
1739   /* Assign workers */
1740   if (sm->num_workers > 1)
1741     {
1742       /* *INDENT-OFF* */
1743       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1744       /* *INDENT-ON* */
1745     }
1746
1747   return 0;
1748 }
1749
1750 int
1751 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1752                   u8 twice_nat)
1753 {
1754   snat_address_t *a = 0;
1755   snat_session_t *ses;
1756   u32 *ses_to_be_removed = 0, *ses_index;
1757   snat_main_per_thread_data_t *tsm;
1758   snat_static_mapping_t *m;
1759   snat_interface_t *interface;
1760   int i;
1761   snat_address_t *addresses =
1762     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1763
1764   /* Find SNAT address */
1765   for (i = 0; i < vec_len (addresses); i++)
1766     {
1767       if (addresses[i].addr.as_u32 == addr.as_u32)
1768         {
1769           a = addresses + i;
1770           break;
1771         }
1772     }
1773   if (!a)
1774     return VNET_API_ERROR_NO_SUCH_ENTRY;
1775
1776   if (delete_sm)
1777     {
1778       /* *INDENT-OFF* */
1779       pool_foreach (m, sm->static_mappings,
1780       ({
1781           if (m->external_addr.as_u32 == addr.as_u32)
1782             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1783                                             m->local_port, m->external_port,
1784                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1785                                             m->proto, 0, m->twice_nat,
1786                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1787       }));
1788       /* *INDENT-ON* */
1789     }
1790   else
1791     {
1792       /* Check if address is used in some static mapping */
1793       if (is_snat_address_used_in_static_mapping (sm, addr))
1794         {
1795           nat_elog_notice ("address used in static mapping");
1796           return VNET_API_ERROR_UNSPECIFIED;
1797         }
1798     }
1799
1800   if (a->fib_index != ~0)
1801     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1802
1803   /* Delete sessions using address */
1804   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1805     {
1806       /* *INDENT-OFF* */
1807       vec_foreach (tsm, sm->per_thread_data)
1808         {
1809           pool_foreach (ses, tsm->sessions, ({
1810             if (ses->out2in.addr.as_u32 == addr.as_u32)
1811               {
1812                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1813                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1814               }
1815           }));
1816
1817           vec_foreach (ses_index, ses_to_be_removed)
1818             {
1819               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1820               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1821             }
1822
1823           vec_free (ses_to_be_removed);
1824         }
1825       /* *INDENT-ON* */
1826     }
1827
1828 #define _(N, i, n, s) \
1829   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1830   vec_free (a->busy_##n##_ports_per_thread);
1831   foreach_snat_protocol
1832 #undef _
1833     if (twice_nat)
1834     {
1835       vec_del1 (sm->twice_nat_addresses, i);
1836       return 0;
1837     }
1838   else
1839     vec_del1 (sm->addresses, i);
1840
1841   /* Delete external address from FIB */
1842   /* *INDENT-OFF* */
1843   pool_foreach (interface, sm->interfaces,
1844   ({
1845     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1846       continue;
1847
1848     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1849     break;
1850   }));
1851   pool_foreach (interface, sm->output_feature_interfaces,
1852   ({
1853     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1854       continue;
1855
1856     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1857     break;
1858   }));
1859   /* *INDENT-ON* */
1860
1861   return 0;
1862 }
1863
1864 int
1865 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1866 {
1867   snat_main_t *sm = &snat_main;
1868   snat_interface_t *i;
1869   const char *feature_name, *del_feature_name;
1870   snat_address_t *ap;
1871   snat_static_mapping_t *m;
1872   snat_det_map_t *dm;
1873   nat_outside_fib_t *outside_fib;
1874   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1875                                                        sw_if_index);
1876
1877   if (sm->out2in_dpo && !is_inside)
1878     return VNET_API_ERROR_UNSUPPORTED;
1879
1880   /* *INDENT-OFF* */
1881   pool_foreach (i, sm->output_feature_interfaces,
1882   ({
1883     if (i->sw_if_index == sw_if_index)
1884       return VNET_API_ERROR_VALUE_EXIST;
1885   }));
1886   /* *INDENT-ON* */
1887
1888   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1889     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1890   else
1891     {
1892       if (sm->num_workers > 1 && !sm->deterministic)
1893         feature_name =
1894           is_inside ? "nat44-in2out-worker-handoff" :
1895           "nat44-out2in-worker-handoff";
1896       else if (sm->deterministic)
1897         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1898       else if (sm->endpoint_dependent)
1899         {
1900           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1901         }
1902       else
1903         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1904     }
1905
1906   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1907     sm->fq_in2out_index =
1908       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1909
1910   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1911     sm->fq_out2in_index =
1912       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1913
1914   if (!is_inside)
1915     {
1916       /* *INDENT-OFF* */
1917       vec_foreach (outside_fib, sm->outside_fibs)
1918         {
1919           if (outside_fib->fib_index == fib_index)
1920             {
1921               if (is_del)
1922                 {
1923                   outside_fib->refcount--;
1924                   if (!outside_fib->refcount)
1925                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1926                 }
1927               else
1928                 outside_fib->refcount++;
1929               goto feature_set;
1930             }
1931         }
1932       /* *INDENT-ON* */
1933       if (!is_del)
1934         {
1935           vec_add2 (sm->outside_fibs, outside_fib, 1);
1936           outside_fib->refcount = 1;
1937           outside_fib->fib_index = fib_index;
1938         }
1939     }
1940 feature_set:
1941   /* *INDENT-OFF* */
1942   pool_foreach (i, sm->interfaces,
1943   ({
1944     if (i->sw_if_index == sw_if_index)
1945       {
1946         if (is_del)
1947           {
1948             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1949               {
1950                 if (is_inside)
1951                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1952                 else
1953                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1954
1955                 if (sm->num_workers > 1 && !sm->deterministic)
1956                   {
1957                     del_feature_name = "nat44-handoff-classify";
1958                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1959                                                  "nat44-out2in-worker-handoff";
1960                   }
1961                 else if (sm->deterministic)
1962                   {
1963                     del_feature_name = "nat44-det-classify";
1964                     feature_name = !is_inside ?  "nat44-det-in2out" :
1965                                                  "nat44-det-out2in";
1966                   }
1967                 else if (sm->endpoint_dependent)
1968                   {
1969                     del_feature_name = "nat44-ed-classify";
1970                     feature_name = !is_inside ?  "nat-pre-in2out" :
1971                                                  "nat-pre-out2in";
1972                   }
1973                 else
1974                   {
1975                     del_feature_name = "nat44-classify";
1976                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1977                   }
1978
1979                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1980                 if (rv)
1981                   return rv;
1982                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1983                                              sw_if_index, 0, 0, 0);
1984                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1985                                              sw_if_index, 1, 0, 0);
1986                 if (!is_inside)
1987                   {
1988                     if (sm->endpoint_dependent)
1989                       vnet_feature_enable_disable ("ip4-local",
1990                                                    "nat44-ed-hairpinning",
1991                                                    sw_if_index, 1, 0, 0);
1992                     else if (!sm->deterministic)
1993                       vnet_feature_enable_disable ("ip4-local",
1994                                                    "nat44-hairpinning",
1995                                                    sw_if_index, 1, 0, 0);
1996                   }
1997               }
1998             else
1999               {
2000                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2001                 if (rv)
2002                   return rv;
2003                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2004                                              sw_if_index, 0, 0, 0);
2005                 pool_put (sm->interfaces, i);
2006                 if (is_inside)
2007                   {
2008                     if (sm->endpoint_dependent)
2009                       vnet_feature_enable_disable ("ip4-local",
2010                                                    "nat44-ed-hairpinning",
2011                                                    sw_if_index, 0, 0, 0);
2012                     else if (!sm->deterministic)
2013                       vnet_feature_enable_disable ("ip4-local",
2014                                                    "nat44-hairpinning",
2015                                                    sw_if_index, 0, 0, 0);
2016                   }
2017               }
2018           }
2019         else
2020           {
2021             if ((nat_interface_is_inside(i) && is_inside) ||
2022                 (nat_interface_is_outside(i) && !is_inside))
2023               return 0;
2024
2025             if (sm->num_workers > 1 && !sm->deterministic)
2026               {
2027                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2028                                                  "nat44-out2in-worker-handoff";
2029                 feature_name = "nat44-handoff-classify";
2030               }
2031             else if (sm->deterministic)
2032               {
2033                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
2034                                                  "nat44-det-out2in";
2035                 feature_name = "nat44-det-classify";
2036               }
2037             else if (sm->endpoint_dependent)
2038               {
2039                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2040                                                  "nat-pre-out2in";
2041
2042                 feature_name = "nat44-ed-classify";
2043               }
2044             else
2045               {
2046                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2047                 feature_name = "nat44-classify";
2048               }
2049
2050             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2051             if (rv)
2052               return rv;
2053             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2054                                          sw_if_index, 0, 0, 0);
2055             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2056                                          sw_if_index, 1, 0, 0);
2057             if (!is_inside)
2058               {
2059                 if (sm->endpoint_dependent)
2060                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2061                                                sw_if_index, 0, 0, 0);
2062                 else if (!sm->deterministic)
2063                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2064                                                sw_if_index, 0, 0, 0);
2065               }
2066             goto set_flags;
2067           }
2068
2069         goto fib;
2070       }
2071   }));
2072   /* *INDENT-ON* */
2073
2074   if (is_del)
2075     return VNET_API_ERROR_NO_SUCH_ENTRY;
2076
2077   pool_get (sm->interfaces, i);
2078   i->sw_if_index = sw_if_index;
2079   i->flags = 0;
2080   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2081                                0);
2082
2083   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2084   if (rv)
2085     return rv;
2086
2087   if (is_inside && !sm->out2in_dpo)
2088     {
2089       if (sm->endpoint_dependent)
2090         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2091                                      sw_if_index, 1, 0, 0);
2092       else if (!sm->deterministic)
2093         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2094                                      sw_if_index, 1, 0, 0);
2095     }
2096
2097 set_flags:
2098   if (is_inside)
2099     {
2100       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2101       return 0;
2102     }
2103   else
2104     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2105
2106   /* Add/delete external addresses to FIB */
2107 fib:
2108   /* *INDENT-OFF* */
2109   vec_foreach (ap, sm->addresses)
2110     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2111
2112   pool_foreach (m, sm->static_mappings,
2113   ({
2114     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2115       continue;
2116
2117     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2118   }));
2119
2120   pool_foreach (dm, sm->det_maps,
2121   ({
2122     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2123   }));
2124   /* *INDENT-ON* */
2125
2126   return 0;
2127 }
2128
2129 int
2130 snat_interface_add_del_output_feature (u32 sw_if_index,
2131                                        u8 is_inside, int is_del)
2132 {
2133   snat_main_t *sm = &snat_main;
2134   snat_interface_t *i;
2135   snat_address_t *ap;
2136   snat_static_mapping_t *m;
2137   nat_outside_fib_t *outside_fib;
2138   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2139                                                        sw_if_index);
2140
2141
2142   if (sm->deterministic ||
2143       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2144     return VNET_API_ERROR_UNSUPPORTED;
2145
2146   /* *INDENT-OFF* */
2147   pool_foreach (i, sm->interfaces,
2148   ({
2149     if (i->sw_if_index == sw_if_index)
2150       return VNET_API_ERROR_VALUE_EXIST;
2151   }));
2152   /* *INDENT-ON* */
2153
2154   if (!is_inside)
2155     {
2156       /* *INDENT-OFF* */
2157       vec_foreach (outside_fib, sm->outside_fibs)
2158         {
2159           if (outside_fib->fib_index == fib_index)
2160             {
2161               if (is_del)
2162                 {
2163                   outside_fib->refcount--;
2164                   if (!outside_fib->refcount)
2165                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2166                 }
2167               else
2168                 outside_fib->refcount++;
2169               goto feature_set;
2170             }
2171         }
2172       /* *INDENT-ON* */
2173       if (!is_del)
2174         {
2175           vec_add2 (sm->outside_fibs, outside_fib, 1);
2176           outside_fib->refcount = 1;
2177           outside_fib->fib_index = fib_index;
2178         }
2179     }
2180
2181 feature_set:
2182   if (is_inside)
2183     {
2184       if (sm->endpoint_dependent)
2185         {
2186           int rv =
2187             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2188           if (rv)
2189             return rv;
2190           rv =
2191             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2192                                                             !is_del);
2193           if (rv)
2194             return rv;
2195           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2196                                        sw_if_index, !is_del, 0, 0);
2197           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2198                                        sw_if_index, !is_del, 0, 0);
2199         }
2200       else
2201         {
2202           int rv =
2203             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2204           if (rv)
2205             return rv;
2206           rv =
2207             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2208                                                             !is_del);
2209           if (rv)
2210             return rv;
2211           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2212                                        sw_if_index, !is_del, 0, 0);
2213           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2214                                        sw_if_index, !is_del, 0, 0);
2215         }
2216       goto fq;
2217     }
2218
2219   if (sm->num_workers > 1)
2220     {
2221       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2222       if (rv)
2223         return rv;
2224       rv =
2225         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2226       if (rv)
2227         return rv;
2228       vnet_feature_enable_disable ("ip4-unicast",
2229                                    "nat44-out2in-worker-handoff",
2230                                    sw_if_index, !is_del, 0, 0);
2231       vnet_feature_enable_disable ("ip4-output",
2232                                    "nat44-in2out-output-worker-handoff",
2233                                    sw_if_index, !is_del, 0, 0);
2234     }
2235   else
2236     {
2237       if (sm->endpoint_dependent)
2238         {
2239           int rv =
2240             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2241           if (rv)
2242             return rv;
2243           rv =
2244             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2245                                                             !is_del);
2246           if (rv)
2247             return rv;
2248           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2249                                        sw_if_index, !is_del, 0, 0);
2250           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2251                                        sw_if_index, !is_del, 0, 0);
2252         }
2253       else
2254         {
2255           int rv =
2256             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2257           if (rv)
2258             return rv;
2259           rv =
2260             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2261                                                             !is_del);
2262           if (rv)
2263             return rv;
2264           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2265                                        sw_if_index, !is_del, 0, 0);
2266           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2267                                        sw_if_index, !is_del, 0, 0);
2268         }
2269     }
2270
2271 fq:
2272   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2273     sm->fq_in2out_output_index =
2274       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2275
2276   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2277     sm->fq_out2in_index =
2278       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2279
2280   /* *INDENT-OFF* */
2281   pool_foreach (i, sm->output_feature_interfaces,
2282   ({
2283     if (i->sw_if_index == sw_if_index)
2284       {
2285         if (is_del)
2286           pool_put (sm->output_feature_interfaces, i);
2287         else
2288           return VNET_API_ERROR_VALUE_EXIST;
2289
2290         goto fib;
2291       }
2292   }));
2293   /* *INDENT-ON* */
2294
2295   if (is_del)
2296     return VNET_API_ERROR_NO_SUCH_ENTRY;
2297
2298   pool_get (sm->output_feature_interfaces, i);
2299   i->sw_if_index = sw_if_index;
2300   i->flags = 0;
2301   if (is_inside)
2302     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2303   else
2304     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2305
2306   /* Add/delete external addresses to FIB */
2307 fib:
2308   if (is_inside)
2309     return 0;
2310
2311   /* *INDENT-OFF* */
2312   vec_foreach (ap, sm->addresses)
2313     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2314
2315   pool_foreach (m, sm->static_mappings,
2316   ({
2317     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2318       continue;
2319
2320     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2321   }));
2322   /* *INDENT-ON* */
2323
2324   return 0;
2325 }
2326
2327 int
2328 snat_set_workers (uword * bitmap)
2329 {
2330   snat_main_t *sm = &snat_main;
2331   int i, j = 0;
2332
2333   if (sm->num_workers < 2)
2334     return VNET_API_ERROR_FEATURE_DISABLED;
2335
2336   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2337     return VNET_API_ERROR_INVALID_WORKER;
2338
2339   vec_free (sm->workers);
2340   /* *INDENT-OFF* */
2341   clib_bitmap_foreach (i, bitmap,
2342     ({
2343       vec_add1(sm->workers, i);
2344       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2345       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2346       j++;
2347     }));
2348   /* *INDENT-ON* */
2349
2350   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2351   sm->num_snat_thread = _vec_len (sm->workers);
2352
2353   return 0;
2354 }
2355
2356 static void
2357 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2358                          u32 old_fib_index)
2359 {
2360   snat_main_t *sm = &snat_main;
2361   nat_outside_fib_t *outside_fib;
2362   snat_interface_t *i;
2363   u8 is_add = 1;
2364   u8 match = 0;
2365
2366   if (new_fib_index == old_fib_index)
2367     return;
2368
2369   if (!vec_len (sm->outside_fibs))
2370     return;
2371
2372   /* *INDENT-OFF* */
2373   pool_foreach (i, sm->interfaces,
2374     ({
2375       if (i->sw_if_index == sw_if_index)
2376         {
2377           if (!(nat_interface_is_outside (i)))
2378             return;
2379           match = 1;
2380         }
2381     }));
2382
2383   pool_foreach (i, sm->output_feature_interfaces,
2384     ({
2385       if (i->sw_if_index == sw_if_index)
2386         {
2387           if (!(nat_interface_is_outside (i)))
2388             return;
2389           match = 1;
2390         }
2391     }));
2392   /* *INDENT-ON* */
2393
2394   if (!match)
2395     return;
2396
2397   vec_foreach (outside_fib, sm->outside_fibs)
2398   {
2399     if (outside_fib->fib_index == old_fib_index)
2400       {
2401         outside_fib->refcount--;
2402         if (!outside_fib->refcount)
2403           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2404         break;
2405       }
2406   }
2407
2408   vec_foreach (outside_fib, sm->outside_fibs)
2409   {
2410     if (outside_fib->fib_index == new_fib_index)
2411       {
2412         outside_fib->refcount++;
2413         is_add = 0;
2414         break;
2415       }
2416   }
2417
2418   if (is_add)
2419     {
2420       vec_add2 (sm->outside_fibs, outside_fib, 1);
2421       outside_fib->refcount = 1;
2422       outside_fib->fib_index = new_fib_index;
2423     }
2424 }
2425
2426 static void
2427 snat_ip4_table_bind (ip4_main_t * im,
2428                      uword opaque,
2429                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2430 {
2431   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2432 }
2433
2434 static void
2435 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2436                                        uword opaque,
2437                                        u32 sw_if_index,
2438                                        ip4_address_t * address,
2439                                        u32 address_length,
2440                                        u32 if_address_index, u32 is_delete);
2441
2442 static void
2443 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2444                                  uword opaque,
2445                                  u32 sw_if_index,
2446                                  ip4_address_t * address,
2447                                  u32 address_length,
2448                                  u32 if_address_index, u32 is_delete);
2449
2450 static int
2451 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2452                                  u32 fib_index,
2453                                  u32 thread_index,
2454                                  snat_session_key_t * k,
2455                                  u16 port_per_thread, u32 snat_thread_index);
2456
2457 static clib_error_t *
2458 snat_init (vlib_main_t * vm)
2459 {
2460   snat_main_t *sm = &snat_main;
2461   clib_error_t *error = 0;
2462   ip4_main_t *im = &ip4_main;
2463   ip_lookup_main_t *lm = &im->lookup_main;
2464   uword *p;
2465   vlib_thread_registration_t *tr;
2466   vlib_thread_main_t *tm = vlib_get_thread_main ();
2467   uword *bitmap = 0;
2468   u32 i;
2469   ip4_add_del_interface_address_callback_t cb4;
2470   vlib_node_t *node;
2471
2472   sm->vlib_main = vm;
2473   sm->vnet_main = vnet_get_main ();
2474   sm->ip4_main = im;
2475   sm->ip4_lookup_main = lm;
2476   sm->api_main = vlibapi_get_main ();
2477   sm->first_worker_index = 0;
2478   sm->num_workers = 0;
2479   sm->num_snat_thread = 1;
2480   sm->workers = 0;
2481   sm->port_per_thread = 0xffff - 1024;
2482   sm->fq_in2out_index = ~0;
2483   sm->fq_in2out_output_index = ~0;
2484   sm->fq_out2in_index = ~0;
2485
2486   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2487   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2488   sm->forwarding_enabled = 0;
2489   sm->log_class = vlib_log_register_class ("nat", 0);
2490   sm->log_level = SNAT_LOG_ERROR;
2491   sm->mss_clamping = 0;
2492
2493   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2494   sm->error_node_index = node->index;
2495
2496   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2497   sm->pre_in2out_node_index = node->index;
2498   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2499   sm->pre_out2in_node_index = node->index;
2500
2501   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2502   sm->pre_in2out_node_index = node->index;
2503
2504   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2505   sm->pre_out2in_node_index = node->index;
2506
2507   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2508   sm->in2out_node_index = node->index;
2509   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2510   sm->in2out_output_node_index = node->index;
2511   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2512   sm->in2out_fast_node_index = node->index;
2513   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2514   sm->in2out_slowpath_node_index = node->index;
2515   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2516   sm->in2out_slowpath_output_node_index = node->index;
2517
2518   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2519   sm->ed_in2out_node_index = node->index;
2520   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2521   sm->ed_in2out_slowpath_node_index = node->index;
2522
2523   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2524   sm->out2in_node_index = node->index;
2525   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2526   sm->out2in_fast_node_index = node->index;
2527
2528   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2529   sm->ed_out2in_node_index = node->index;
2530   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2531   sm->ed_out2in_slowpath_node_index = node->index;
2532
2533   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2534   sm->det_in2out_node_index = node->index;
2535   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2536   sm->det_out2in_node_index = node->index;
2537
2538   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2539   sm->hairpinning_node_index = node->index;
2540   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2541   sm->hairpin_dst_node_index = node->index;
2542   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2543   sm->hairpin_src_node_index = node->index;
2544   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2545   sm->ed_hairpinning_node_index = node->index;
2546   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2547   sm->ed_hairpin_dst_node_index = node->index;
2548   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2549   sm->ed_hairpin_src_node_index = node->index;
2550
2551   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2552   if (p)
2553     {
2554       tr = (vlib_thread_registration_t *) p[0];
2555       if (tr)
2556         {
2557           sm->num_workers = tr->count;
2558           sm->first_worker_index = tr->first_index;
2559         }
2560     }
2561
2562   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2563
2564   /* Use all available workers by default */
2565   if (sm->num_workers > 1)
2566     {
2567       for (i = 0; i < sm->num_workers; i++)
2568         bitmap = clib_bitmap_set (bitmap, i, 1);
2569       snat_set_workers (bitmap);
2570       clib_bitmap_free (bitmap);
2571     }
2572   else
2573     {
2574       sm->per_thread_data[0].snat_thread_index = 0;
2575     }
2576
2577   error = snat_api_init (vm, sm);
2578   if (error)
2579     return error;
2580
2581   /* Set up the interface address add/del callback */
2582   cb4.function = snat_ip4_add_del_interface_address_cb;
2583   cb4.function_opaque = 0;
2584
2585   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2586
2587   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2588   cb4.function_opaque = 0;
2589
2590   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2591
2592   nat_dpo_module_init ();
2593
2594   /* Init counters */
2595   sm->total_users.name = "total-users";
2596   sm->total_users.stat_segment_name = "/nat44/total-users";
2597   vlib_validate_simple_counter (&sm->total_users, 0);
2598   vlib_zero_simple_counter (&sm->total_users, 0);
2599   sm->total_sessions.name = "total-sessions";
2600   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2601   vlib_validate_simple_counter (&sm->total_sessions, 0);
2602   vlib_zero_simple_counter (&sm->total_sessions, 0);
2603
2604   /* Init IPFIX logging */
2605   snat_ipfix_logging_init (vm);
2606
2607   /* Init NAT64 */
2608   error = nat64_init (vm);
2609   if (error)
2610     return error;
2611
2612   nat66_init (vm);
2613
2614   ip4_table_bind_callback_t cbt4 = {
2615     .function = snat_ip4_table_bind,
2616   };
2617   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2618
2619   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2620                                         FIB_SOURCE_PRIORITY_HI,
2621                                         FIB_SOURCE_BH_SIMPLE);
2622   nat_fib_src_low = fib_source_allocate ("nat-low",
2623                                          FIB_SOURCE_PRIORITY_LOW,
2624                                          FIB_SOURCE_BH_SIMPLE);
2625
2626   return error;
2627 }
2628
2629 VLIB_INIT_FUNCTION (snat_init);
2630
2631 void
2632 snat_free_outside_address_and_port (snat_address_t * addresses,
2633                                     u32 thread_index, snat_session_key_t * k)
2634 {
2635   snat_address_t *a;
2636   u32 address_index;
2637   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2638
2639   for (address_index = 0; address_index < vec_len (addresses);
2640        address_index++)
2641     {
2642       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2643         break;
2644     }
2645
2646   ASSERT (address_index < vec_len (addresses));
2647
2648   a = addresses + address_index;
2649
2650   switch (k->protocol)
2651     {
2652 #define _(N, i, n, s) \
2653     case SNAT_PROTOCOL_##N: \
2654       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2655         port_host_byte_order) == 1); \
2656       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2657         port_host_byte_order, 0); \
2658       a->busy_##n##_ports--; \
2659       a->busy_##n##_ports_per_thread[thread_index]--; \
2660       break;
2661       foreach_snat_protocol
2662 #undef _
2663     default:
2664       nat_elog_info ("unknown protocol");
2665       return;
2666     }
2667 }
2668
2669 static int
2670 nat_set_outside_address_and_port (snat_address_t * addresses,
2671                                   u32 thread_index, snat_session_key_t * k)
2672 {
2673   snat_address_t *a = 0;
2674   u32 address_index;
2675   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2676
2677   for (address_index = 0; address_index < vec_len (addresses);
2678        address_index++)
2679     {
2680       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2681         continue;
2682
2683       a = addresses + address_index;
2684       switch (k->protocol)
2685         {
2686 #define _(N, j, n, s) \
2687         case SNAT_PROTOCOL_##N: \
2688           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2689             return VNET_API_ERROR_INSTANCE_IN_USE; \
2690           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2691           a->busy_##n##_ports_per_thread[thread_index]++; \
2692           a->busy_##n##_ports++; \
2693           return 0;
2694           foreach_snat_protocol
2695 #undef _
2696         default:
2697           nat_elog_info ("unknown protocol");
2698           return 1;
2699         }
2700     }
2701
2702   return VNET_API_ERROR_NO_SUCH_ENTRY;
2703 }
2704
2705 int
2706 snat_static_mapping_match (snat_main_t * sm,
2707                            snat_session_key_t match,
2708                            snat_session_key_t * mapping,
2709                            u8 by_external,
2710                            u8 * is_addr_only,
2711                            twice_nat_type_t * twice_nat,
2712                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2713                            u8 * is_identity_nat)
2714 {
2715   clib_bihash_kv_8_8_t kv, value;
2716   snat_static_mapping_t *m;
2717   snat_session_key_t m_key;
2718   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2719   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2720   u8 backend_index;
2721   nat44_lb_addr_port_t *local;
2722
2723   m_key.fib_index = match.fib_index;
2724   if (by_external)
2725     {
2726       mapping_hash = &sm->static_mapping_by_external;
2727       m_key.fib_index = 0;
2728     }
2729
2730   m_key.addr = match.addr;
2731   m_key.port = clib_net_to_host_u16 (match.port);
2732   m_key.protocol = match.protocol;
2733
2734   kv.key = m_key.as_u64;
2735
2736   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2737     {
2738       /* Try address only mapping */
2739       m_key.port = 0;
2740       m_key.protocol = 0;
2741       kv.key = m_key.as_u64;
2742       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2743         return 1;
2744     }
2745
2746   m = pool_elt_at_index (sm->static_mappings, value.value);
2747
2748   if (by_external)
2749     {
2750       if (is_lb_static_mapping (m))
2751         {
2752           if (PREDICT_FALSE (lb != 0))
2753             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2754           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2755                                                           match.addr,
2756                                                           match.protocol,
2757                                                           match.port,
2758                                                           &backend_index))
2759             {
2760               local = pool_elt_at_index (m->locals, backend_index);
2761               mapping->addr = local->addr;
2762               mapping->port = clib_host_to_net_u16 (local->port);
2763               mapping->fib_index = local->fib_index;
2764               goto end;
2765             }
2766           // pick locals matching this worker
2767           if (PREDICT_FALSE (sm->num_workers > 1))
2768             {
2769               u32 thread_index = vlib_get_thread_index ();
2770               /* *INDENT-OFF* */
2771               pool_foreach_index (i, m->locals,
2772               ({
2773                 local = pool_elt_at_index (m->locals, i);
2774
2775                 ip4_header_t ip = {
2776                   .src_address = local->addr,
2777                 };
2778
2779                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2780                     thread_index)
2781                   {
2782                     vec_add1 (tmp, i);
2783                   }
2784               }));
2785               /* *INDENT-ON* */
2786               ASSERT (vec_len (tmp) != 0);
2787             }
2788           else
2789             {
2790               /* *INDENT-OFF* */
2791               pool_foreach_index (i, m->locals,
2792               ({
2793                 vec_add1 (tmp, i);
2794               }));
2795               /* *INDENT-ON* */
2796             }
2797           hi = vec_len (tmp) - 1;
2798           local = pool_elt_at_index (m->locals, tmp[hi]);
2799           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2800           while (lo < hi)
2801             {
2802               mid = ((hi - lo) >> 1) + lo;
2803               local = pool_elt_at_index (m->locals, tmp[mid]);
2804               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2805             }
2806           local = pool_elt_at_index (m->locals, tmp[lo]);
2807           if (!(local->prefix >= rand))
2808             return 1;
2809           mapping->addr = local->addr;
2810           mapping->port = clib_host_to_net_u16 (local->port);
2811           mapping->fib_index = local->fib_index;
2812           if (m->affinity)
2813             {
2814               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2815                                                 match.protocol, match.port,
2816                                                 tmp[lo], m->affinity,
2817                                                 m->affinity_per_service_list_head_index))
2818                 nat_elog_info ("create affinity record failed");
2819             }
2820           vec_free (tmp);
2821         }
2822       else
2823         {
2824           if (PREDICT_FALSE (lb != 0))
2825             *lb = NO_LB_NAT;
2826           mapping->fib_index = m->fib_index;
2827           mapping->addr = m->local_addr;
2828           /* Address only mapping doesn't change port */
2829           mapping->port = is_addr_only_static_mapping (m) ? match.port
2830             : clib_host_to_net_u16 (m->local_port);
2831         }
2832       mapping->protocol = m->proto;
2833     }
2834   else
2835     {
2836       mapping->addr = m->external_addr;
2837       /* Address only mapping doesn't change port */
2838       mapping->port = is_addr_only_static_mapping (m) ? match.port
2839         : clib_host_to_net_u16 (m->external_port);
2840       mapping->fib_index = sm->outside_fib_index;
2841     }
2842
2843 end:
2844   if (PREDICT_FALSE (is_addr_only != 0))
2845     *is_addr_only = is_addr_only_static_mapping (m);
2846
2847   if (PREDICT_FALSE (twice_nat != 0))
2848     *twice_nat = m->twice_nat;
2849
2850   if (PREDICT_FALSE (is_identity_nat != 0))
2851     *is_identity_nat = is_identity_static_mapping (m);
2852
2853   return 0;
2854 }
2855
2856 static_always_inline u16
2857 snat_random_port (u16 min, u16 max)
2858 {
2859   snat_main_t *sm = &snat_main;
2860   return min + random_u32 (&sm->random_seed) /
2861     (random_u32_max () / (max - min + 1) + 1);
2862 }
2863
2864 int
2865 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2866                                      u32 fib_index,
2867                                      u32 thread_index,
2868                                      snat_session_key_t * k,
2869                                      u16 port_per_thread,
2870                                      u32 snat_thread_index)
2871 {
2872   snat_main_t *sm = &snat_main;
2873
2874   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2875                                   port_per_thread, snat_thread_index);
2876 }
2877
2878 static int
2879 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2880                                  u32 fib_index,
2881                                  u32 thread_index,
2882                                  snat_session_key_t * k,
2883                                  u16 port_per_thread, u32 snat_thread_index)
2884 {
2885   int i;
2886   snat_address_t *a, *ga = 0;
2887   u32 portnum;
2888
2889   for (i = 0; i < vec_len (addresses); i++)
2890     {
2891       a = addresses + i;
2892       switch (k->protocol)
2893         {
2894 #define _(N, j, n, s) \
2895         case SNAT_PROTOCOL_##N: \
2896           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2897             { \
2898               if (a->fib_index == fib_index) \
2899                 { \
2900                   while (1) \
2901                     { \
2902                       portnum = (port_per_thread * \
2903                         snat_thread_index) + \
2904                         snat_random_port(1, port_per_thread) + 1024; \
2905                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2906                         continue; \
2907                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2908                       a->busy_##n##_ports_per_thread[thread_index]++; \
2909                       a->busy_##n##_ports++; \
2910                       k->addr = a->addr; \
2911                       k->port = clib_host_to_net_u16(portnum); \
2912                       return 0; \
2913                     } \
2914                 } \
2915               else if (a->fib_index == ~0) \
2916                 { \
2917                   ga = a; \
2918                 } \
2919             } \
2920           break;
2921           foreach_snat_protocol
2922 #undef _
2923         default:
2924           nat_elog_info ("unknown protocol");
2925           return 1;
2926         }
2927
2928     }
2929
2930   if (ga)
2931     {
2932       a = ga;
2933       switch (k->protocol)
2934         {
2935 #define _(N, j, n, s) \
2936         case SNAT_PROTOCOL_##N: \
2937           while (1) \
2938             { \
2939               portnum = (port_per_thread * \
2940                 snat_thread_index) + \
2941                 snat_random_port(1, port_per_thread) + 1024; \
2942               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2943                 continue; \
2944               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2945               a->busy_##n##_ports_per_thread[thread_index]++; \
2946               a->busy_##n##_ports++; \
2947               k->addr = a->addr; \
2948               k->port = clib_host_to_net_u16(portnum); \
2949               return 0; \
2950             }
2951           break;
2952           foreach_snat_protocol
2953 #undef _
2954         default:
2955           nat_elog_info ("unknown protocol");
2956           return 1;
2957         }
2958     }
2959
2960   /* Totally out of translations to use... */
2961   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2962   return 1;
2963 }
2964
2965 static int
2966 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2967                               u32 fib_index,
2968                               u32 thread_index,
2969                               snat_session_key_t * k,
2970                               u16 port_per_thread, u32 snat_thread_index)
2971 {
2972   snat_main_t *sm = &snat_main;
2973   snat_address_t *a = addresses;
2974   u16 m, ports, portnum, A, j;
2975   m = 16 - (sm->psid_offset + sm->psid_length);
2976   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2977
2978   if (!vec_len (addresses))
2979     goto exhausted;
2980
2981   switch (k->protocol)
2982     {
2983 #define _(N, i, n, s) \
2984     case SNAT_PROTOCOL_##N: \
2985       if (a->busy_##n##_ports < ports) \
2986         { \
2987           while (1) \
2988             { \
2989               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2990               j = snat_random_port(0, pow2_mask(m)); \
2991               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2992               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2993                 continue; \
2994               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2995               a->busy_##n##_ports++; \
2996               k->addr = a->addr; \
2997               k->port = clib_host_to_net_u16 (portnum); \
2998               return 0; \
2999             } \
3000         } \
3001       break;
3002       foreach_snat_protocol
3003 #undef _
3004     default:
3005       nat_elog_info ("unknown protocol");
3006       return 1;
3007     }
3008
3009 exhausted:
3010   /* Totally out of translations to use... */
3011   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3012   return 1;
3013 }
3014
3015 static int
3016 nat_alloc_addr_and_port_range (snat_address_t * addresses,
3017                                u32 fib_index,
3018                                u32 thread_index,
3019                                snat_session_key_t * k,
3020                                u16 port_per_thread, u32 snat_thread_index)
3021 {
3022   snat_main_t *sm = &snat_main;
3023   snat_address_t *a = addresses;
3024   u16 portnum, ports;
3025
3026   ports = sm->end_port - sm->start_port + 1;
3027
3028   if (!vec_len (addresses))
3029     goto exhausted;
3030
3031   switch (k->protocol)
3032     {
3033 #define _(N, i, n, s) \
3034     case SNAT_PROTOCOL_##N: \
3035       if (a->busy_##n##_ports < ports) \
3036         { \
3037           while (1) \
3038             { \
3039               portnum = snat_random_port(sm->start_port, sm->end_port); \
3040               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
3041                 continue; \
3042               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
3043               a->busy_##n##_ports++; \
3044               k->addr = a->addr; \
3045               k->port = clib_host_to_net_u16 (portnum); \
3046               return 0; \
3047             } \
3048         } \
3049       break;
3050       foreach_snat_protocol
3051 #undef _
3052     default:
3053       nat_elog_info ("unknown protocol");
3054       return 1;
3055     }
3056
3057 exhausted:
3058   /* Totally out of translations to use... */
3059   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3060   return 1;
3061 }
3062
3063 void
3064 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3065 {
3066   dpo_id_t dpo_v4 = DPO_INVALID;
3067   fib_prefix_t pfx = {
3068     .fp_proto = FIB_PROTOCOL_IP4,
3069     .fp_len = 32,
3070     .fp_addr.ip4.as_u32 = addr.as_u32,
3071   };
3072
3073   if (is_add)
3074     {
3075       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3076       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3077                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3078       dpo_reset (&dpo_v4);
3079     }
3080   else
3081     {
3082       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3083     }
3084 }
3085
3086 u8 *
3087 format_session_kvp (u8 * s, va_list * args)
3088 {
3089   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3090   snat_session_key_t k;
3091
3092   k.as_u64 = v->key;
3093
3094   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
3095
3096   return s;
3097 }
3098
3099 u8 *
3100 format_static_mapping_kvp (u8 * s, va_list * args)
3101 {
3102   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3103   snat_session_key_t k;
3104
3105   k.as_u64 = v->key;
3106
3107   s = format (s, "%U static-mapping-index %llu",
3108               format_static_mapping_key, &k, v->value);
3109
3110   return s;
3111 }
3112
3113 u8 *
3114 format_user_kvp (u8 * s, va_list * args)
3115 {
3116   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3117   snat_user_key_t k;
3118
3119   k.as_u64 = v->key;
3120
3121   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3122               k.fib_index, v->value);
3123
3124   return s;
3125 }
3126
3127 u8 *
3128 format_ed_session_kvp (u8 * s, va_list * args)
3129 {
3130   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3131   nat_ed_ses_key_t k;
3132
3133   k.as_u64[0] = v->key[0];
3134   k.as_u64[1] = v->key[1];
3135
3136   s =
3137     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3138             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
3139             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
3140             format_ip_protocol, k.proto, k.fib_index, v->value);
3141
3142   return s;
3143 }
3144
3145 static u32
3146 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3147                            u8 is_output)
3148 {
3149   snat_main_t *sm = &snat_main;
3150   u32 next_worker_index = 0;
3151   u32 hash;
3152
3153   next_worker_index = sm->first_worker_index;
3154   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3155     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3156
3157   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3158     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3159   else
3160     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3161
3162   return next_worker_index;
3163 }
3164
3165 static u32
3166 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3167                            u32 rx_fib_index0, u8 is_output)
3168 {
3169   snat_main_t *sm = &snat_main;
3170   udp_header_t *udp;
3171   u16 port;
3172   snat_session_key_t m_key;
3173   clib_bihash_kv_8_8_t kv, value;
3174   snat_static_mapping_t *m;
3175   u32 proto;
3176   u32 next_worker_index = 0;
3177
3178   /* first try static mappings without port */
3179   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3180     {
3181       m_key.addr = ip0->dst_address;
3182       m_key.port = 0;
3183       m_key.protocol = 0;
3184       m_key.fib_index = rx_fib_index0;
3185       kv.key = m_key.as_u64;
3186       if (!clib_bihash_search_8_8
3187           (&sm->static_mapping_by_external, &kv, &value))
3188         {
3189           m = pool_elt_at_index (sm->static_mappings, value.value);
3190           return m->workers[0];
3191         }
3192     }
3193
3194   proto = ip_proto_to_snat_proto (ip0->protocol);
3195   udp = ip4_next_header (ip0);
3196   port = udp->dst_port;
3197
3198   /* unknown protocol */
3199   if (PREDICT_FALSE (proto == ~0))
3200     {
3201       /* use current thread */
3202       return vlib_get_thread_index ();
3203     }
3204
3205   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3206     {
3207       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3208       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3209       if (!icmp_type_is_error_message
3210           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3211         port = vnet_buffer (b)->ip.reass.l4_src_port;
3212       else
3213         {
3214           /* if error message, then it's not fragmented and we can access it */
3215           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3216           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3217           void *l4_header = ip4_next_header (inner_ip);
3218           switch (proto)
3219             {
3220             case SNAT_PROTOCOL_ICMP:
3221               icmp = (icmp46_header_t *) l4_header;
3222               echo = (icmp_echo_header_t *) (icmp + 1);
3223               port = echo->identifier;
3224               break;
3225             case SNAT_PROTOCOL_UDP:
3226             case SNAT_PROTOCOL_TCP:
3227               port = ((tcp_udp_header_t *) l4_header)->src_port;
3228               break;
3229             default:
3230               return vlib_get_thread_index ();
3231             }
3232         }
3233     }
3234
3235   /* try static mappings with port */
3236   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3237     {
3238       m_key.addr = ip0->dst_address;
3239       m_key.port = clib_net_to_host_u16 (port);
3240       m_key.protocol = proto;
3241       m_key.fib_index = rx_fib_index0;
3242       kv.key = m_key.as_u64;
3243       if (!clib_bihash_search_8_8
3244           (&sm->static_mapping_by_external, &kv, &value))
3245         {
3246           m = pool_elt_at_index (sm->static_mappings, value.value);
3247           return m->workers[0];
3248         }
3249     }
3250
3251   /* worker by outside port */
3252   next_worker_index = sm->first_worker_index;
3253   next_worker_index +=
3254     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3255   return next_worker_index;
3256 }
3257
3258 static u32
3259 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3260                                u8 is_output)
3261 {
3262   snat_main_t *sm = &snat_main;
3263   u32 next_worker_index = sm->first_worker_index;
3264   u32 hash;
3265
3266   clib_bihash_kv_16_8_t kv16, value16;
3267   snat_main_per_thread_data_t *tsm;
3268   udp_header_t *udp;
3269
3270   if (PREDICT_FALSE (is_output))
3271     {
3272       u32 fib_index = sm->outside_fib_index;
3273       nat_outside_fib_t *outside_fib;
3274       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3275       fib_prefix_t pfx = {
3276         .fp_proto = FIB_PROTOCOL_IP4,
3277         .fp_len = 32,
3278         .fp_addr = {
3279                     .ip4.as_u32 = ip->dst_address.as_u32,
3280                     }
3281         ,
3282       };
3283
3284       udp = ip4_next_header (ip);
3285
3286       switch (vec_len (sm->outside_fibs))
3287         {
3288         case 0:
3289           fib_index = sm->outside_fib_index;
3290           break;
3291         case 1:
3292           fib_index = sm->outside_fibs[0].fib_index;
3293           break;
3294         default:
3295             /* *INDENT-OFF* */
3296             vec_foreach (outside_fib, sm->outside_fibs)
3297               {
3298                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3299                 if (FIB_NODE_INDEX_INVALID != fei)
3300                   {
3301                     if (fib_entry_get_resolving_interface (fei) != ~0)
3302                       {
3303                         fib_index = outside_fib->fib_index;
3304                         break;
3305                       }
3306                   }
3307               }
3308             /* *INDENT-ON* */
3309           break;
3310         }
3311
3312       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3313                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3314
3315       /* *INDENT-OFF* */
3316       vec_foreach (tsm, sm->per_thread_data)
3317         {
3318           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3319                                                       &kv16, &value16)))
3320             {
3321               next_worker_index += tsm->thread_index;
3322
3323               nat_elog_debug_handoff (
3324                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3325                 next_worker_index, fib_index,
3326                 clib_net_to_host_u32 (ip->src_address.as_u32),
3327                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3328
3329               return next_worker_index;
3330             }
3331         }
3332       /* *INDENT-ON* */
3333     }
3334
3335   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3336     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3337
3338   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3339     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3340   else
3341     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3342
3343   if (PREDICT_TRUE (!is_output))
3344     {
3345       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3346                               next_worker_index, rx_fib_index,
3347                               clib_net_to_host_u32 (ip->src_address.as_u32),
3348                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3349     }
3350   else
3351     {
3352       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3353                               next_worker_index, rx_fib_index,
3354                               clib_net_to_host_u32 (ip->src_address.as_u32),
3355                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3356     }
3357
3358   return next_worker_index;
3359 }
3360
3361 static u32
3362 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3363                                u32 rx_fib_index, u8 is_output)
3364 {
3365   snat_main_t *sm = &snat_main;
3366   clib_bihash_kv_8_8_t kv, value;
3367   clib_bihash_kv_16_8_t kv16, value16;
3368   snat_main_per_thread_data_t *tsm;
3369
3370   u32 proto, next_worker_index = 0;
3371   udp_header_t *udp;
3372   u16 port;
3373   snat_static_mapping_t *m;
3374   u32 hash;
3375
3376   proto = ip_proto_to_snat_proto (ip->protocol);
3377
3378   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3379     {
3380       udp = ip4_next_header (ip);
3381
3382       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3383                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3384
3385       /* *INDENT-OFF* */
3386       vec_foreach (tsm, sm->per_thread_data)
3387         {
3388           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3389                                                       &kv16, &value16)))
3390             {
3391               next_worker_index = sm->first_worker_index + tsm->thread_index;
3392               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3393                           next_worker_index, rx_fib_index,
3394                           clib_net_to_host_u32 (ip->src_address.as_u32),
3395                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3396               return next_worker_index;
3397             }
3398           }
3399         /* *INDENT-ON* */
3400     }
3401   else if (proto == SNAT_PROTOCOL_ICMP)
3402     {
3403       nat_ed_ses_key_t key;
3404
3405       if (!get_icmp_o2i_ed_key (b, ip, &key))
3406         {
3407
3408           key.fib_index = rx_fib_index;
3409           kv16.key[0] = key.as_u64[0];
3410           kv16.key[1] = key.as_u64[1];
3411
3412           /* *INDENT-OFF* */
3413           vec_foreach (tsm, sm->per_thread_data)
3414             {
3415               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3416                                                           &kv16, &value16)))
3417                 {
3418                   next_worker_index = sm->first_worker_index +
3419                                       tsm->thread_index;
3420                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3421                               next_worker_index, rx_fib_index,
3422                               clib_net_to_host_u32 (ip->src_address.as_u32),
3423                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3424                   return next_worker_index;
3425                 }
3426             }
3427           /* *INDENT-ON* */
3428         }
3429     }
3430
3431   /* first try static mappings without port */
3432   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3433     {
3434       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3435       if (!clib_bihash_search_8_8
3436           (&sm->static_mapping_by_external, &kv, &value))
3437         {
3438           m = pool_elt_at_index (sm->static_mappings, value.value);
3439           next_worker_index = m->workers[0];
3440           goto done;
3441         }
3442     }
3443
3444   /* unknown protocol */
3445   if (PREDICT_FALSE (proto == ~0))
3446     {
3447       /* use current thread */
3448       next_worker_index = vlib_get_thread_index ();
3449       goto done;
3450     }
3451
3452   udp = ip4_next_header (ip);
3453   port = udp->dst_port;
3454
3455   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3456     {
3457       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3458       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3459       if (!icmp_type_is_error_message
3460           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3461         port = vnet_buffer (b)->ip.reass.l4_src_port;
3462       else
3463         {
3464           /* if error message, then it's not fragmented and we can access it */
3465           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3466           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3467           void *l4_header = ip4_next_header (inner_ip);
3468           switch (proto)
3469             {
3470             case SNAT_PROTOCOL_ICMP:
3471               icmp = (icmp46_header_t *) l4_header;
3472               echo = (icmp_echo_header_t *) (icmp + 1);
3473               port = echo->identifier;
3474               break;
3475             case SNAT_PROTOCOL_UDP:
3476             case SNAT_PROTOCOL_TCP:
3477               port = ((tcp_udp_header_t *) l4_header)->src_port;
3478               break;
3479             default:
3480               next_worker_index = vlib_get_thread_index ();
3481               goto done;
3482             }
3483         }
3484     }
3485
3486   /* try static mappings with port */
3487   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3488     {
3489       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3490                   clib_net_to_host_u16 (port));
3491       if (!clib_bihash_search_8_8
3492           (&sm->static_mapping_by_external, &kv, &value))
3493         {
3494           m = pool_elt_at_index (sm->static_mappings, value.value);
3495           if (!is_lb_static_mapping (m))
3496             {
3497               next_worker_index = m->workers[0];
3498               goto done;
3499             }
3500
3501           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3502             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3503
3504           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3505             next_worker_index =
3506               m->workers[hash & (_vec_len (m->workers) - 1)];
3507           else
3508             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3509           goto done;
3510         }
3511     }
3512
3513   /* worker by outside port */
3514   next_worker_index = sm->first_worker_index;
3515   next_worker_index +=
3516     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3517
3518 done:
3519   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3520                           clib_net_to_host_u32 (ip->src_address.as_u32),
3521                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3522   return next_worker_index;
3523 }
3524
3525 void
3526 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3527                 ip4_address_t * out_addr, u16 out_port,
3528                 ip4_address_t * eh_addr, u16 eh_port,
3529                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3530                 u32 fib_index, u16 flags, u32 thread_index)
3531 {
3532   snat_main_t *sm = &snat_main;
3533   snat_session_key_t key;
3534   snat_user_t *u;
3535   snat_session_t *s;
3536   clib_bihash_kv_8_8_t kv;
3537   f64 now = vlib_time_now (sm->vlib_main);
3538   nat_outside_fib_t *outside_fib;
3539   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3540   snat_main_per_thread_data_t *tsm;
3541   fib_prefix_t pfx = {
3542     .fp_proto = FIB_PROTOCOL_IP4,
3543     .fp_len = 32,
3544     .fp_addr = {
3545                 .ip4.as_u32 = eh_addr->as_u32,
3546                 },
3547   };
3548
3549   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3550
3551   key.addr.as_u32 = out_addr->as_u32;
3552   key.port = out_port;
3553   key.protocol = proto;
3554
3555   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3556     {
3557       if (nat_set_outside_address_and_port
3558           (sm->addresses, thread_index, &key))
3559         return;
3560     }
3561
3562   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3563   if (!u)
3564     return;
3565
3566   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3567   if (!s)
3568     return;
3569
3570   s->last_heard = now;
3571   s->flags = flags;
3572   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3573   s->ext_host_port = eh_port;
3574   user_session_increment (sm, u, snat_is_session_static (s));
3575   switch (vec_len (sm->outside_fibs))
3576     {
3577     case 0:
3578       key.fib_index = sm->outside_fib_index;
3579       break;
3580     case 1:
3581       key.fib_index = sm->outside_fibs[0].fib_index;
3582       break;
3583     default:
3584       /* *INDENT-OFF* */
3585       vec_foreach (outside_fib, sm->outside_fibs)
3586         {
3587           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3588           if (FIB_NODE_INDEX_INVALID != fei)
3589             {
3590               if (fib_entry_get_resolving_interface (fei) != ~0)
3591                 {
3592                   key.fib_index = outside_fib->fib_index;
3593                   break;
3594                 }
3595             }
3596         }
3597       /* *INDENT-ON* */
3598       break;
3599     }
3600   s->out2in = key;
3601   kv.key = key.as_u64;
3602   kv.value = s - tsm->sessions;
3603   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3604     nat_elog_warn ("out2in key add failed");
3605
3606   key.addr.as_u32 = in_addr->as_u32;
3607   key.port = in_port;
3608   key.fib_index = fib_index;
3609   s->in2out = key;
3610   kv.key = key.as_u64;
3611   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3612     nat_elog_warn ("in2out key add failed");
3613 }
3614
3615 void
3616 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3617                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3618                 u32 ti)
3619 {
3620   snat_main_t *sm = &snat_main;
3621   snat_session_key_t key;
3622   clib_bihash_kv_8_8_t kv, value;
3623   u32 thread_index;
3624   snat_session_t *s;
3625   snat_main_per_thread_data_t *tsm;
3626
3627   if (sm->num_workers > 1)
3628     thread_index =
3629       sm->first_worker_index +
3630       (sm->workers[(clib_net_to_host_u16 (out_port) -
3631                     1024) / sm->port_per_thread]);
3632   else
3633     thread_index = sm->num_workers;
3634   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3635
3636   key.addr.as_u32 = out_addr->as_u32;
3637   key.port = out_port;
3638   key.protocol = proto;
3639   key.fib_index = fib_index;
3640   kv.key = key.as_u64;
3641   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3642     return;
3643
3644   s = pool_elt_at_index (tsm->sessions, value.value);
3645   nat_free_session_data (sm, s, thread_index, 1);
3646   nat44_delete_session (sm, s, thread_index);
3647 }
3648
3649 void
3650 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3651                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3652                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3653 {
3654   snat_main_t *sm = &snat_main;
3655   snat_session_key_t key;
3656   clib_bihash_kv_8_8_t kv, value;
3657   snat_session_t *s;
3658   snat_main_per_thread_data_t *tsm;
3659
3660   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3661
3662   key.addr.as_u32 = out_addr->as_u32;
3663   key.port = out_port;
3664   key.protocol = proto;
3665   key.fib_index = fib_index;
3666   kv.key = key.as_u64;
3667   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3668     return;
3669
3670   s = pool_elt_at_index (tsm->sessions, value.value);
3671   s->total_pkts = total_pkts;
3672   s->total_bytes = total_bytes;
3673 }
3674
3675 void
3676 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3677                    ip4_address_t * out_addr, u16 out_port,
3678                    ip4_address_t * eh_addr, u16 eh_port,
3679                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3680                    u32 fib_index, u16 flags, u32 thread_index)
3681 {
3682   snat_main_t *sm = &snat_main;
3683   snat_session_key_t key;
3684   snat_user_t *u;
3685   snat_session_t *s;
3686   clib_bihash_kv_16_8_t kv;
3687   f64 now = vlib_time_now (sm->vlib_main);
3688   nat_outside_fib_t *outside_fib;
3689   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3690   snat_main_per_thread_data_t *tsm;
3691   fib_prefix_t pfx = {
3692     .fp_proto = FIB_PROTOCOL_IP4,
3693     .fp_len = 32,
3694     .fp_addr = {
3695                 .ip4.as_u32 = eh_addr->as_u32,
3696                 },
3697   };
3698
3699   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3700
3701   key.addr.as_u32 = out_addr->as_u32;
3702   key.port = out_port;
3703   key.protocol = proto;
3704
3705   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3706     {
3707       if (nat_set_outside_address_and_port
3708           (sm->addresses, thread_index, &key))
3709         return;
3710     }
3711
3712   key.addr.as_u32 = ehn_addr->as_u32;
3713   key.port = ehn_port;
3714   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3715     {
3716       if (nat_set_outside_address_and_port
3717           (sm->twice_nat_addresses, thread_index, &key))
3718         return;
3719     }
3720
3721   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3722   if (!u)
3723     return;
3724
3725   s = nat_ed_session_alloc (sm, u, thread_index, now);
3726   if (!s)
3727     return;
3728
3729   s->last_heard = now;
3730   s->flags = flags;
3731   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3732   s->ext_host_nat_port = s->ext_host_port = eh_port;
3733   if (is_twice_nat_session (s))
3734     {
3735       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3736       s->ext_host_nat_port = ehn_port;
3737     }
3738   user_session_increment (sm, u, snat_is_session_static (s));
3739   switch (vec_len (sm->outside_fibs))
3740     {
3741     case 0:
3742       key.fib_index = sm->outside_fib_index;
3743       break;
3744     case 1:
3745       key.fib_index = sm->outside_fibs[0].fib_index;
3746       break;
3747     default:
3748       /* *INDENT-OFF* */
3749       vec_foreach (outside_fib, sm->outside_fibs)
3750         {
3751           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3752           if (FIB_NODE_INDEX_INVALID != fei)
3753             {
3754               if (fib_entry_get_resolving_interface (fei) != ~0)
3755                 {
3756                   key.fib_index = outside_fib->fib_index;
3757                   break;
3758                 }
3759             }
3760         }
3761       /* *INDENT-ON* */
3762       break;
3763     }
3764   key.addr.as_u32 = out_addr->as_u32;
3765   key.port = out_port;
3766   s->out2in = key;
3767   kv.value = s - tsm->sessions;
3768
3769   key.addr.as_u32 = in_addr->as_u32;
3770   key.port = in_port;
3771   key.fib_index = fib_index;
3772   s->in2out = key;
3773
3774   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3775               snat_proto_to_ip_proto (proto), fib_index, in_port,
3776               s->ext_host_nat_port);
3777   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3778     nat_elog_warn ("in2out key add failed");
3779
3780   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3781               s->out2in.fib_index, out_port, eh_port);
3782   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3783     nat_elog_warn ("out2in key add failed");
3784 }
3785
3786 void
3787 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3788                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3789                    u32 fib_index, u32 ti)
3790 {
3791   snat_main_t *sm = &snat_main;
3792   nat_ed_ses_key_t key;
3793   clib_bihash_kv_16_8_t kv, value;
3794   u32 thread_index;
3795   snat_session_t *s;
3796   snat_main_per_thread_data_t *tsm;
3797
3798   if (sm->num_workers > 1)
3799     thread_index =
3800       sm->first_worker_index +
3801       (sm->workers[(clib_net_to_host_u16 (out_port) -
3802                     1024) / sm->port_per_thread]);
3803   else
3804     thread_index = sm->num_workers;
3805   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3806
3807   key.l_addr.as_u32 = out_addr->as_u32;
3808   key.l_port = out_port;
3809   key.r_addr.as_u32 = eh_addr->as_u32;
3810   key.r_port = eh_port;
3811   key.proto = proto;
3812   key.fib_index = fib_index;
3813   kv.key[0] = key.as_u64[0];
3814   kv.key[1] = key.as_u64[1];
3815   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3816     return;
3817
3818   s = pool_elt_at_index (tsm->sessions, value.value);
3819   nat_free_session_data (sm, s, thread_index, 1);
3820   nat44_delete_session (sm, s, thread_index);
3821 }
3822
3823 void
3824 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3825                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3826                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3827                    u32 thread_index)
3828 {
3829   snat_main_t *sm = &snat_main;
3830   nat_ed_ses_key_t key;
3831   clib_bihash_kv_16_8_t kv, value;
3832   snat_session_t *s;
3833   snat_main_per_thread_data_t *tsm;
3834
3835   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3836
3837   key.l_addr.as_u32 = out_addr->as_u32;
3838   key.l_port = out_port;
3839   key.r_addr.as_u32 = eh_addr->as_u32;
3840   key.r_port = eh_port;
3841   key.proto = proto;
3842   key.fib_index = fib_index;
3843   kv.key[0] = key.as_u64[0];
3844   kv.key[1] = key.as_u64[1];
3845   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3846     return;
3847
3848   s = pool_elt_at_index (tsm->sessions, value.value);
3849   s->total_pkts = total_pkts;
3850   s->total_bytes = total_bytes;
3851 }
3852
3853 static clib_error_t *
3854 snat_config (vlib_main_t * vm, unformat_input_t * input)
3855 {
3856   snat_main_t *sm = &snat_main;
3857   nat66_main_t *nm = &nat66_main;
3858   //dslite_main_t *dm = &dslite_main;
3859   snat_main_per_thread_data_t *tsm;
3860
3861   u32 static_mapping_buckets = 1024;
3862   u32 static_mapping_memory_size = 64 << 20;
3863
3864   u32 nat64_bib_buckets = 1024;
3865   u32 nat64_bib_memory_size = 128 << 20;
3866
3867   u32 nat64_st_buckets = 2048;
3868   u32 nat64_st_memory_size = 256 << 20;
3869
3870   u32 user_buckets = 128;
3871   u32 user_memory_size = 64 << 20;
3872   u32 translation_buckets = 1024;
3873   u32 translation_memory_size = 128 << 20;
3874
3875   u32 max_translations_per_user = ~0;
3876
3877   u32 outside_vrf_id = 0;
3878   u32 outside_ip6_vrf_id = 0;
3879   u32 inside_vrf_id = 0;
3880   u8 static_mapping_only = 0;
3881   u8 static_mapping_connection_tracking = 0;
3882
3883   // configurable timeouts
3884   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3885   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3886   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3887   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3888
3889   sm->deterministic = 0;
3890   sm->out2in_dpo = 0;
3891   sm->endpoint_dependent = 0;
3892
3893   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3894     {
3895       if (unformat
3896           (input, "translation hash buckets %d", &translation_buckets))
3897         ;
3898       else if (unformat (input, "udp timeout %d", &udp_timeout))
3899         ;
3900       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3901         ;
3902       else if (unformat (input, "tcp transitory timeout %d",
3903                          &tcp_transitory_timeout));
3904       else if (unformat (input, "tcp established timeout %d",
3905                          &tcp_established_timeout));
3906       else if (unformat (input, "translation hash memory %d",
3907                          &translation_memory_size));
3908       else if (unformat (input, "user hash buckets %d", &user_buckets))
3909         ;
3910       else if (unformat (input, "user hash memory %d", &user_memory_size))
3911         ;
3912       else if (unformat (input, "max translations per user %d",
3913                          &max_translations_per_user))
3914         ;
3915       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3916         ;
3917       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3918         ;
3919       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3920         ;
3921       else if (unformat (input, "static mapping only"))
3922         {
3923           static_mapping_only = 1;
3924           if (unformat (input, "connection tracking"))
3925             static_mapping_connection_tracking = 1;
3926         }
3927       else if (unformat (input, "deterministic"))
3928         sm->deterministic = 1;
3929       else if (unformat (input, "nat64 bib hash buckets %d",
3930                          &nat64_bib_buckets))
3931         ;
3932       else if (unformat (input, "nat64 bib hash memory %d",
3933                          &nat64_bib_memory_size))
3934         ;
3935       else
3936         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3937         ;
3938       else if (unformat (input, "nat64 st hash memory %d",
3939                          &nat64_st_memory_size))
3940         ;
3941       else if (unformat (input, "out2in dpo"))
3942         sm->out2in_dpo = 1;
3943       //else if (unformat (input, "dslite ce"))
3944       //dslite_set_ce (dm, 1);
3945       else if (unformat (input, "endpoint-dependent"))
3946         sm->endpoint_dependent = 1;
3947       else
3948         return clib_error_return (0, "unknown input '%U'",
3949                                   format_unformat_error, input);
3950     }
3951
3952   if (sm->deterministic && sm->endpoint_dependent)
3953     return clib_error_return (0,
3954                               "deterministic and endpoint-dependent modes are mutually exclusive");
3955
3956   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3957     return clib_error_return (0,
3958                               "static mapping only mode available only for simple nat");
3959
3960   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3961     return clib_error_return (0,
3962                               "out2in dpo mode available only for simple nat");
3963
3964   /* optionally configurable timeouts for testing purposes */
3965   sm->udp_timeout = udp_timeout;
3966   sm->tcp_transitory_timeout = tcp_transitory_timeout;
3967   sm->tcp_established_timeout = tcp_established_timeout;
3968   sm->icmp_timeout = icmp_timeout;
3969
3970   sm->min_timeout = nat44_minimal_timeout (sm);
3971
3972   sm->user_buckets = user_buckets;
3973   sm->user_memory_size = user_memory_size;
3974
3975   sm->translation_buckets = translation_buckets;
3976   sm->translation_memory_size = translation_memory_size;
3977
3978   /* do not exceed load factor 10 */
3979   sm->max_translations = 10 * translation_buckets;
3980   sm->max_translations_per_user = max_translations_per_user == ~0 ?
3981     sm->max_translations : max_translations_per_user;
3982
3983   sm->outside_vrf_id = outside_vrf_id;
3984   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3985                                                              outside_vrf_id,
3986                                                              nat_fib_src_hi);
3987   nm->outside_vrf_id = outside_ip6_vrf_id;
3988   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3989                                                              outside_ip6_vrf_id,
3990                                                              nat_fib_src_hi);
3991   sm->inside_vrf_id = inside_vrf_id;
3992   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3993                                                             inside_vrf_id,
3994                                                             nat_fib_src_hi);
3995   sm->static_mapping_only = static_mapping_only;
3996   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3997
3998   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3999                   nat64_st_memory_size);
4000
4001   if (sm->deterministic)
4002     {
4003       sm->in2out_node_index = snat_det_in2out_node.index;
4004       sm->in2out_output_node_index = ~0;
4005       sm->out2in_node_index = snat_det_out2in_node.index;
4006       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4007       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4008     }
4009   else
4010     {
4011       if (sm->endpoint_dependent)
4012         {
4013           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4014           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4015
4016           sm->handoff_out2in_index = nat_pre_out2in_node.index;
4017           sm->handoff_in2out_index = nat_pre_in2out_node.index;
4018           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
4019
4020           sm->in2out_node_index = nat44_ed_in2out_node.index;
4021           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4022           sm->out2in_node_index = nat44_ed_out2in_node.index;
4023
4024           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4025           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4026           nat_affinity_init (vm);
4027           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4028                        nat_ha_sref_ed_cb);
4029         }
4030       else
4031         {
4032           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4033           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4034
4035           sm->handoff_out2in_index = snat_out2in_node.index;
4036           sm->handoff_in2out_index = snat_in2out_node.index;
4037           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
4038
4039           sm->in2out_node_index = snat_in2out_node.index;
4040           sm->in2out_output_node_index = snat_in2out_output_node.index;
4041           sm->out2in_node_index = snat_out2in_node.index;
4042           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4043           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4044           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4045         }
4046       if (!static_mapping_only ||
4047           (static_mapping_only && static_mapping_connection_tracking))
4048         {
4049           /* *INDENT-OFF* */
4050           vec_foreach (tsm, sm->per_thread_data)
4051             {
4052               tsm->min_session_timeout = 0;
4053
4054               tsm->cleared = 0;
4055               tsm->cleanup_runs = 0;
4056               tsm->cleanup_timeout = 0;
4057
4058               if (sm->endpoint_dependent)
4059                 {
4060                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
4061                                          translation_buckets,
4062                                          translation_memory_size);
4063                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
4064                                                       format_ed_session_kvp);
4065
4066                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
4067                                          translation_buckets,
4068                                          translation_memory_size);
4069                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
4070                                                       format_ed_session_kvp);
4071                 }
4072               else
4073                 {
4074                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
4075                                         translation_buckets,
4076                                         translation_memory_size);
4077                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
4078                                                      format_session_kvp);
4079
4080                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
4081                                         translation_buckets,
4082                                         translation_memory_size);
4083                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
4084                                                      format_session_kvp);
4085                 }
4086
4087               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
4088                                     user_memory_size);
4089               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
4090                                                  format_user_kvp);
4091             }
4092           /* *INDENT-ON* */
4093
4094         }
4095       else
4096         {
4097           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4098           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4099         }
4100       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4101                             "static_mapping_by_local", static_mapping_buckets,
4102                             static_mapping_memory_size);
4103       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4104                                          format_static_mapping_kvp);
4105
4106       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4107                             "static_mapping_by_external",
4108                             static_mapping_buckets,
4109                             static_mapping_memory_size);
4110       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4111                                          format_static_mapping_kvp);
4112     }
4113
4114   return 0;
4115 }
4116
4117 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4118
4119 static void
4120 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4121                                  uword opaque,
4122                                  u32 sw_if_index,
4123                                  ip4_address_t * address,
4124                                  u32 address_length,
4125                                  u32 if_address_index, u32 is_delete)
4126 {
4127   snat_main_t *sm = &snat_main;
4128   snat_static_map_resolve_t *rp;
4129   snat_static_mapping_t *m;
4130   snat_session_key_t m_key;
4131   clib_bihash_kv_8_8_t kv, value;
4132   int i, rv;
4133   ip4_address_t l_addr;
4134
4135   for (i = 0; i < vec_len (sm->to_resolve); i++)
4136     {
4137       rp = sm->to_resolve + i;
4138       if (rp->addr_only == 0)
4139         continue;
4140       if (rp->sw_if_index == sw_if_index)
4141         goto match;
4142     }
4143
4144   return;
4145
4146 match:
4147   m_key.addr.as_u32 = address->as_u32;
4148   m_key.port = rp->addr_only ? 0 : rp->e_port;
4149   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4150   m_key.fib_index = sm->outside_fib_index;
4151   kv.key = m_key.as_u64;
4152   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4153     m = 0;
4154   else
4155     m = pool_elt_at_index (sm->static_mappings, value.value);
4156
4157   if (!is_delete)
4158     {
4159       /* Don't trip over lease renewal, static config */
4160       if (m)
4161         return;
4162     }
4163   else
4164     {
4165       if (!m)
4166         return;
4167     }
4168
4169   /* Indetity mapping? */
4170   if (rp->l_addr.as_u32 == 0)
4171     l_addr.as_u32 = address[0].as_u32;
4172   else
4173     l_addr.as_u32 = rp->l_addr.as_u32;
4174   /* Add the static mapping */
4175   rv = snat_add_static_mapping (l_addr,
4176                                 address[0],
4177                                 rp->l_port,
4178                                 rp->e_port,
4179                                 rp->vrf_id,
4180                                 rp->addr_only, ~0 /* sw_if_index */ ,
4181                                 rp->proto, !is_delete, rp->twice_nat,
4182                                 rp->out2in_only, rp->tag, rp->identity_nat);
4183   if (rv)
4184     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4185 }
4186
4187 static void
4188 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4189                                        uword opaque,
4190                                        u32 sw_if_index,
4191                                        ip4_address_t * address,
4192                                        u32 address_length,
4193                                        u32 if_address_index, u32 is_delete)
4194 {
4195   snat_main_t *sm = &snat_main;
4196   snat_static_map_resolve_t *rp;
4197   ip4_address_t l_addr;
4198   int i, j;
4199   int rv;
4200   u8 twice_nat = 0;
4201   snat_address_t *addresses = sm->addresses;
4202
4203   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4204     {
4205       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4206         goto match;
4207     }
4208
4209   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4210     {
4211       twice_nat = 1;
4212       addresses = sm->twice_nat_addresses;
4213       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4214         goto match;
4215     }
4216
4217   return;
4218
4219 match:
4220   if (!is_delete)
4221     {
4222       /* Don't trip over lease renewal, static config */
4223       for (j = 0; j < vec_len (addresses); j++)
4224         if (addresses[j].addr.as_u32 == address->as_u32)
4225           return;
4226
4227       (void) snat_add_address (sm, address, ~0, twice_nat);
4228       /* Scan static map resolution vector */
4229       for (j = 0; j < vec_len (sm->to_resolve); j++)
4230         {
4231           rp = sm->to_resolve + j;
4232           if (rp->addr_only)
4233             continue;
4234           /* On this interface? */
4235           if (rp->sw_if_index == sw_if_index)
4236             {
4237               /* Indetity mapping? */
4238               if (rp->l_addr.as_u32 == 0)
4239                 l_addr.as_u32 = address[0].as_u32;
4240               else
4241                 l_addr.as_u32 = rp->l_addr.as_u32;
4242               /* Add the static mapping */
4243               rv = snat_add_static_mapping (l_addr,
4244                                             address[0],
4245                                             rp->l_port,
4246                                             rp->e_port,
4247                                             rp->vrf_id,
4248                                             rp->addr_only,
4249                                             ~0 /* sw_if_index */ ,
4250                                             rp->proto,
4251                                             rp->is_add, rp->twice_nat,
4252                                             rp->out2in_only, rp->tag,
4253                                             rp->identity_nat);
4254               if (rv)
4255                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4256                                     "i4", rv);
4257             }
4258         }
4259       return;
4260     }
4261   else
4262     {
4263       (void) snat_del_address (sm, address[0], 1, twice_nat);
4264       return;
4265     }
4266 }
4267
4268
4269 int
4270 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4271                             u8 twice_nat)
4272 {
4273   ip4_main_t *ip4_main = sm->ip4_main;
4274   ip4_address_t *first_int_addr;
4275   snat_static_map_resolve_t *rp;
4276   u32 *indices_to_delete = 0;
4277   int i, j;
4278   u32 *auto_add_sw_if_indices =
4279     twice_nat ? sm->
4280     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4281
4282   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4283     );
4284
4285   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4286     {
4287       if (auto_add_sw_if_indices[i] == sw_if_index)
4288         {
4289           if (is_del)
4290             {
4291               /* if have address remove it */
4292               if (first_int_addr)
4293                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4294               else
4295                 {
4296                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4297                     {
4298                       rp = sm->to_resolve + j;
4299                       if (rp->sw_if_index == sw_if_index)
4300                         vec_add1 (indices_to_delete, j);
4301                     }
4302                   if (vec_len (indices_to_delete))
4303                     {
4304                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4305                         vec_del1 (sm->to_resolve, j);
4306                       vec_free (indices_to_delete);
4307                     }
4308                 }
4309               if (twice_nat)
4310                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4311               else
4312                 vec_del1 (sm->auto_add_sw_if_indices, i);
4313             }
4314           else
4315             return VNET_API_ERROR_VALUE_EXIST;
4316
4317           return 0;
4318         }
4319     }
4320
4321   if (is_del)
4322     return VNET_API_ERROR_NO_SUCH_ENTRY;
4323
4324   /* add to the auto-address list */
4325   if (twice_nat)
4326     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4327   else
4328     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4329
4330   /* If the address is already bound - or static - add it now */
4331   if (first_int_addr)
4332     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4333
4334   return 0;
4335 }
4336
4337 int
4338 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4339                    snat_protocol_t proto, u32 vrf_id, int is_in)
4340 {
4341   snat_main_per_thread_data_t *tsm;
4342   clib_bihash_kv_8_8_t kv, value;
4343   ip4_header_t ip;
4344   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4345   snat_session_key_t key;
4346   snat_session_t *s;
4347   clib_bihash_8_8_t *t;
4348
4349   if (sm->endpoint_dependent)
4350     return VNET_API_ERROR_UNSUPPORTED;
4351
4352   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4353   if (sm->num_workers > 1)
4354     tsm =
4355       vec_elt_at_index (sm->per_thread_data,
4356                         sm->worker_in2out_cb (&ip, fib_index, 0));
4357   else
4358     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4359
4360   key.addr.as_u32 = addr->as_u32;
4361   key.port = clib_host_to_net_u16 (port);
4362   key.protocol = proto;
4363   key.fib_index = fib_index;
4364   kv.key = key.as_u64;
4365   t = is_in ? &tsm->in2out : &tsm->out2in;
4366   if (!clib_bihash_search_8_8 (t, &kv, &value))
4367     {
4368       if (pool_is_free_index (tsm->sessions, value.value))
4369         return VNET_API_ERROR_UNSPECIFIED;
4370
4371       s = pool_elt_at_index (tsm->sessions, value.value);
4372       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4373       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4374       return 0;
4375     }
4376
4377   return VNET_API_ERROR_NO_SUCH_ENTRY;
4378 }
4379
4380 int
4381 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4382                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4383                       u32 vrf_id, int is_in)
4384 {
4385   ip4_header_t ip;
4386   clib_bihash_16_8_t *t;
4387   nat_ed_ses_key_t key;
4388   clib_bihash_kv_16_8_t kv, value;
4389   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4390   snat_session_t *s;
4391   snat_main_per_thread_data_t *tsm;
4392
4393   if (!sm->endpoint_dependent)
4394     return VNET_API_ERROR_FEATURE_DISABLED;
4395
4396   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4397   if (sm->num_workers > 1)
4398     tsm =
4399       vec_elt_at_index (sm->per_thread_data,
4400                         sm->worker_in2out_cb (&ip, fib_index, 0));
4401   else
4402     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4403
4404   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4405   key.l_addr.as_u32 = addr->as_u32;
4406   key.r_addr.as_u32 = eh_addr->as_u32;
4407   key.l_port = clib_host_to_net_u16 (port);
4408   key.r_port = clib_host_to_net_u16 (eh_port);
4409   key.proto = proto;
4410   key.fib_index = fib_index;
4411   kv.key[0] = key.as_u64[0];
4412   kv.key[1] = key.as_u64[1];
4413   if (clib_bihash_search_16_8 (t, &kv, &value))
4414     return VNET_API_ERROR_NO_SUCH_ENTRY;
4415
4416   if (pool_is_free_index (tsm->sessions, value.value))
4417     return VNET_API_ERROR_UNSPECIFIED;
4418   s = pool_elt_at_index (tsm->sessions, value.value);
4419   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4420   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4421   return 0;
4422 }
4423
4424 void
4425 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4426 {
4427   snat_main_t *sm = &snat_main;
4428
4429   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4430   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4431   sm->psid = psid;
4432   sm->psid_offset = psid_offset;
4433   sm->psid_length = psid_length;
4434 }
4435
4436 void
4437 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4438 {
4439   snat_main_t *sm = &snat_main;
4440
4441   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4442   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4443   sm->start_port = start_port;
4444   sm->end_port = end_port;
4445 }
4446
4447 void
4448 nat_set_alloc_addr_and_port_default (void)
4449 {
4450   snat_main_t *sm = &snat_main;
4451
4452   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4453   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4454 }
4455
4456 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4457                                  vlib_node_runtime_t * node,
4458                                  vlib_frame_t * frame)
4459 {
4460   return 0;
4461 }
4462
4463 /* *INDENT-OFF* */
4464 VLIB_REGISTER_NODE (nat_default_node) = {
4465   .name = "nat-default",
4466   .vector_size = sizeof (u32),
4467   .format_trace = 0,
4468   .type = VLIB_NODE_TYPE_INTERNAL,
4469   .n_errors = 0,
4470   .n_next_nodes = NAT_N_NEXT,
4471   .next_nodes = {
4472     [NAT_NEXT_DROP] = "error-drop",
4473     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4474     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4475     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4476     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4477     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4478     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4479     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4480     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4481   },
4482 };
4483 /* *INDENT-ON* */
4484
4485 /*
4486  * fd.io coding-style-patch-verification: ON
4487  *
4488  * Local Variables:
4489  * eval: (c-set-style "gnu")
4490  * End:
4491  */