src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 fib_source_t nat_fib_src_hi;
  42 fib_source_t nat_fib_src_low;
  43
  44 /* *INDENT-OFF* */
  45 /* Hook up input features */
  46 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat-pre-in2out",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-sv-reassembly-feature"),
  51 };
  52 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat-pre-out2in",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  56                                "ip4-dhcp-client-detect",
  57                                "ip4-sv-reassembly-feature"),
  58 };
  59 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-in2out-worker-handoff",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-out2in-worker-handoff",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-in2out",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  74 };
  75 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-out2in",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  79                                "ip4-dhcp-client-detect"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-classify",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  85 };
  86 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-det-in2out",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-det-out2in",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  95                                "ip4-dhcp-client-detect"),
  96 };
  97 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-det-classify",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-ed-in2out",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 106 };
 107 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-ed-out2in",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 111                                "ip4-dhcp-client-detect"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-ed-classify",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-handoff-classify",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 122 };
 123 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-in2out-fast",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 127 };
 128 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 129   .arc_name = "ip4-unicast",
 130   .node_name = "nat44-out2in-fast",
 131   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 132                                "ip4-dhcp-client-detect"),
 133 };
 134 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 135   .arc_name = "ip4-unicast",
 136   .node_name = "nat44-hairpin-dst",
 137   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 140   .arc_name = "ip4-unicast",
 141   .node_name = "nat44-ed-hairpin-dst",
 142   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 143 };
 144
 145 /* Hook up output features */
 146 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-in2out-output",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-in2out-output-worker-handoff",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 155 };
 156 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 157   .arc_name = "ip4-output",
 158   .node_name = "nat44-hairpin-src",
 159   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 162   .arc_name = "ip4-output",
 163   .node_name = "nat44-ed-in2out-output",
 164   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 165   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 166 };
 167 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 168   .arc_name = "ip4-output",
 169   .node_name = "nat44-ed-hairpin-src",
 170   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 171   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 172 };
 173
 174 /* Hook up ip4-local features */
 175 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 176 {
 177   .arc_name = "ip4-local",
 178   .node_name = "nat44-hairpinning",
 179   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 180 };
 181 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 182 {
 183   .arc_name = "ip4-local",
 184   .node_name = "nat44-ed-hairpinning",
 185   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 186 };
 187
 188
 189 VLIB_PLUGIN_REGISTER () = {
 190     .version = VPP_BUILD_VER,
 191     .description = "Network Address Translation (NAT)",
 192 };
 193 /* *INDENT-ON* */
 194
 195 void
 196 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 197                        u8 is_ha)
 198 {
 199   snat_session_key_t key;
 200   clib_bihash_kv_8_8_t kv;
 201   nat_ed_ses_key_t ed_key;
 202   clib_bihash_kv_16_8_t ed_kv;
 203   snat_main_per_thread_data_t *tsm =
 204     vec_elt_at_index (sm->per_thread_data, thread_index);
 205
 206   if (is_fwd_bypass_session (s))
 207     {
 208       if (snat_is_unk_proto_session (s))
 209         {
 210           ed_key.proto = s->in2out.port;
 211           ed_key.r_port = 0;
 212           ed_key.l_port = 0;
 213         }
 214       else
 215         {
 216           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 217           ed_key.l_port = s->in2out.port;
 218           ed_key.r_port = s->ext_host_port;
 219         }
 220       ed_key.l_addr = s->in2out.addr;
 221       ed_key.r_addr = s->ext_host_addr;
 222       ed_key.fib_index = 0;
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 226         nat_elog_warn ("in2out_ed key del failed");
 227       return;
 228     }
 229
 230   /* session lookup tables */
 231   if (is_ed_session (s))
 232     {
 233       if (is_affinity_sessions (s))
 234         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 235                              s->in2out.protocol, s->out2in.port);
 236       ed_key.l_addr = s->out2in.addr;
 237       ed_key.r_addr = s->ext_host_addr;
 238       ed_key.fib_index = s->out2in.fib_index;
 239       if (snat_is_unk_proto_session (s))
 240         {
 241           ed_key.proto = s->in2out.port;
 242           ed_key.r_port = 0;
 243           ed_key.l_port = 0;
 244         }
 245       else
 246         {
 247           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 248           ed_key.l_port = s->out2in.port;
 249           ed_key.r_port = s->ext_host_port;
 250         }
 251       ed_kv.key[0] = ed_key.as_u64[0];
 252       ed_kv.key[1] = ed_key.as_u64[1];
 253       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 254         nat_elog_warn ("out2in_ed key del failed");
 255       ed_key.l_addr = s->in2out.addr;
 256       ed_key.fib_index = s->in2out.fib_index;
 257       if (!snat_is_unk_proto_session (s))
 258         ed_key.l_port = s->in2out.port;
 259       if (is_twice_nat_session (s))
 260         {
 261           ed_key.r_addr = s->ext_host_nat_addr;
 262           ed_key.r_port = s->ext_host_nat_port;
 263         }
 264       ed_kv.key[0] = ed_key.as_u64[0];
 265       ed_kv.key[1] = ed_key.as_u64[1];
 266       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 267         nat_elog_warn ("in2out_ed key del failed");
 268
 269       if (!is_ha)
 270         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 271                                &s->in2out.addr, s->in2out.port,
 272                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 273                                &s->out2in.addr, s->out2in.port,
 274                                &s->ext_host_addr, s->ext_host_port,
 275                                s->in2out.protocol, is_twice_nat_session (s));
 276     }
 277   else
 278     {
 279       kv.key = s->in2out.as_u64;
 280       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 281         nat_elog_warn ("in2out key del failed");
 282       kv.key = s->out2in.as_u64;
 283       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 284         nat_elog_warn ("out2in key del failed");
 285
 286       if (!is_ha)
 287         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 288                                  &s->in2out.addr, s->in2out.port,
 289                                  &s->out2in.addr, s->out2in.port,
 290                                  s->in2out.protocol);
 291     }
 292
 293   if (snat_is_unk_proto_session (s))
 294     return;
 295
 296   if (!is_ha)
 297     {
 298       /* log NAT event */
 299       snat_ipfix_logging_nat44_ses_delete (thread_index,
 300                                            s->in2out.addr.as_u32,
 301                                            s->out2in.addr.as_u32,
 302                                            s->in2out.protocol,
 303                                            s->in2out.port,
 304                                            s->out2in.port,
 305                                            s->in2out.fib_index);
 306
 307       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 308                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 309                    thread_index);
 310     }
 311
 312   /* Twice NAT address and port for external host */
 313   if (is_twice_nat_session (s))
 314     {
 315       key.protocol = s->in2out.protocol;
 316       key.port = s->ext_host_nat_port;
 317       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 318       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 319                                           thread_index, &key);
 320     }
 321
 322   if (snat_is_session_static (s))
 323     return;
 324
 325   snat_free_outside_address_and_port (sm->addresses, thread_index,
 326                                       &s->out2in);
 327 }
 328
 329 void
 330 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 331                          u32 thread_index, u8 is_ha)
 332 {
 333   snat_session_key_t key;
 334   nat_ed_ses_key_t ed_key;
 335   clib_bihash_kv_16_8_t ed_kv;
 336   snat_main_per_thread_data_t *tsm =
 337     vec_elt_at_index (sm->per_thread_data, thread_index);
 338
 339   if (is_fwd_bypass_session (s))
 340     {
 341       if (snat_is_unk_proto_session (s))
 342         {
 343           ed_key.proto = s->in2out.port;
 344           ed_key.r_port = 0;
 345           ed_key.l_port = 0;
 346         }
 347       else
 348         {
 349           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 350           ed_key.l_port = s->in2out.port;
 351           ed_key.r_port = s->ext_host_port;
 352         }
 353
 354       ed_key.l_addr = s->in2out.addr;
 355       ed_key.r_addr = s->ext_host_addr;
 356       ed_key.fib_index = 0;
 357       ed_kv.key[0] = ed_key.as_u64[0];
 358       ed_kv.key[1] = ed_key.as_u64[1];
 359
 360       if (PREDICT_FALSE
 361           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 362         nat_elog_warn ("in2out_ed key del failed");
 363       return;
 364     }
 365
 366   /* session lookup tables */
 367   if (is_affinity_sessions (s))
 368     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 369                          s->in2out.protocol, s->out2in.port);
 370   ed_key.l_addr = s->out2in.addr;
 371   ed_key.r_addr = s->ext_host_addr;
 372   ed_key.fib_index = s->out2in.fib_index;
 373   if (snat_is_unk_proto_session (s))
 374     {
 375       ed_key.proto = s->in2out.port;
 376       ed_key.r_port = 0;
 377       ed_key.l_port = 0;
 378     }
 379   else
 380     {
 381       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 382       ed_key.l_port = s->out2in.port;
 383       ed_key.r_port = s->ext_host_port;
 384     }
 385   ed_kv.key[0] = ed_key.as_u64[0];
 386   ed_kv.key[1] = ed_key.as_u64[1];
 387
 388   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0)))
 389     nat_elog_warn ("out2in_ed key del failed");
 390
 391   ed_key.l_addr = s->in2out.addr;
 392   ed_key.fib_index = s->in2out.fib_index;
 393
 394   if (!snat_is_unk_proto_session (s))
 395     ed_key.l_port = s->in2out.port;
 396
 397   if (is_twice_nat_session (s))
 398     {
 399       ed_key.r_addr = s->ext_host_nat_addr;
 400       ed_key.r_port = s->ext_host_nat_port;
 401     }
 402
 403   ed_kv.key[0] = ed_key.as_u64[0];
 404   ed_kv.key[1] = ed_key.as_u64[1];
 405
 406   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 407     nat_elog_warn ("in2out_ed key del failed");
 408
 409   if (!is_ha)
 410     {
 411       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 412                              &s->in2out.addr, s->in2out.port,
 413                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 414                              &s->out2in.addr, s->out2in.port,
 415                              &s->ext_host_addr, s->ext_host_port,
 416                              s->in2out.protocol, is_twice_nat_session (s));
 417     }
 418
 419   if (snat_is_unk_proto_session (s))
 420     return;
 421
 422   // is this correct ?
 423   if (!is_ha)
 424     {
 425       snat_ipfix_logging_nat44_ses_delete (thread_index,
 426                                            s->in2out.addr.as_u32,
 427                                            s->out2in.addr.as_u32,
 428                                            s->in2out.protocol,
 429                                            s->in2out.port,
 430                                            s->out2in.port,
 431                                            s->in2out.fib_index);
 432       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 433                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 434                    thread_index);
 435     }
 436
 437   /* Twice NAT address and port for external host */
 438   if (is_twice_nat_session (s))
 439     {
 440       key.protocol = s->in2out.protocol;
 441       key.port = s->ext_host_nat_port;
 442       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 443       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 444                                           thread_index, &key);
 445     }
 446
 447   if (snat_is_session_static (s))
 448     return;
 449
 450   // should be called for every dynamic session
 451   snat_free_outside_address_and_port (sm->addresses, thread_index,
 452                                       &s->out2in);
 453 }
 454
 455
 456 snat_user_t *
 457 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 458                         u32 thread_index)
 459 {
 460   snat_user_t *u = 0;
 461   snat_user_key_t user_key;
 462   clib_bihash_kv_8_8_t kv, value;
 463   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 464   dlist_elt_t *per_user_list_head_elt;
 465
 466   user_key.addr.as_u32 = addr->as_u32;
 467   user_key.fib_index = fib_index;
 468   kv.key = user_key.as_u64;
 469
 470   /* Ever heard of the "user" = src ip4 address before? */
 471   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 472     {
 473       /* no, make a new one */
 474       pool_get (tsm->users, u);
 475       clib_memset (u, 0, sizeof (*u));
 476
 477       u->min_session_timeout = 0;
 478
 479       u->addr.as_u32 = addr->as_u32;
 480       u->fib_index = fib_index;
 481
 482       pool_get (tsm->list_pool, per_user_list_head_elt);
 483
 484       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 485         tsm->list_pool;
 486
 487       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 488
 489       kv.value = u - tsm->users;
 490
 491       /* add user */
 492       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 493         nat_elog_warn ("user_hash keay add failed");
 494
 495       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 496                                pool_elts (tsm->users));
 497     }
 498   else
 499     {
 500       u = pool_elt_at_index (tsm->users, value.value);
 501     }
 502
 503   return u;
 504 }
 505
 506 snat_session_t *
 507 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 508                               u32 thread_index, f64 now)
 509 {
 510   snat_session_t *s;
 511   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 512   u32 oldest_per_user_translation_list_index, session_index;
 513   dlist_elt_t *oldest_per_user_translation_list_elt;
 514   dlist_elt_t *per_user_translation_list_elt;
 515
 516   /* Over quota? Recycle the least recently used translation */
 517   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 518     {
 519       oldest_per_user_translation_list_index =
 520         clib_dlist_remove_head (tsm->list_pool,
 521                                 u->sessions_per_user_list_head_index);
 522
 523       ASSERT (oldest_per_user_translation_list_index != ~0);
 524
 525       /* Add it back to the end of the LRU list */
 526       clib_dlist_addtail (tsm->list_pool,
 527                           u->sessions_per_user_list_head_index,
 528                           oldest_per_user_translation_list_index);
 529       /* Get the list element */
 530       oldest_per_user_translation_list_elt =
 531         pool_elt_at_index (tsm->list_pool,
 532                            oldest_per_user_translation_list_index);
 533
 534       /* Get the session index from the list element */
 535       session_index = oldest_per_user_translation_list_elt->value;
 536
 537       /* Get the session */
 538       s = pool_elt_at_index (tsm->sessions, session_index);
 539       nat_free_session_data (sm, s, thread_index, 0);
 540       if (snat_is_session_static (s))
 541         u->nstaticsessions--;
 542       else
 543         u->nsessions--;
 544       s->flags = 0;
 545       s->total_bytes = 0;
 546       s->total_pkts = 0;
 547       s->state = 0;
 548       s->ext_host_addr.as_u32 = 0;
 549       s->ext_host_port = 0;
 550       s->ext_host_nat_addr.as_u32 = 0;
 551       s->ext_host_nat_port = 0;
 552     }
 553   else
 554     {
 555       pool_get (tsm->sessions, s);
 556       clib_memset (s, 0, sizeof (*s));
 557
 558       /* Create list elts */
 559       pool_get (tsm->list_pool, per_user_translation_list_elt);
 560       clib_dlist_init (tsm->list_pool,
 561                        per_user_translation_list_elt - tsm->list_pool);
 562
 563       per_user_translation_list_elt->value = s - tsm->sessions;
 564       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 565       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 566
 567       clib_dlist_addtail (tsm->list_pool,
 568                           s->per_user_list_head_index,
 569                           per_user_translation_list_elt - tsm->list_pool);
 570
 571       s->user_index = u - tsm->users;
 572       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 573                                pool_elts (tsm->sessions));
 574     }
 575
 576   s->ha_last_refreshed = now;
 577
 578   return s;
 579 }
 580
 581 snat_session_t *
 582 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 583                       f64 now)
 584 {
 585   snat_session_t *s;
 586   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 587
 588   dlist_elt_t *oldest_elt;
 589   u64 sess_timeout_time;
 590   u32 oldest_index;
 591
 592   // no sessions
 593   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 594     goto alloc_new;
 595
 596   // no free sessions
 597   if (PREDICT_FALSE
 598       ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user))
 599     {
 600       if (nat44_max_translations_per_user_cleanup (u, thread_index, now))
 601         goto alloc_new;
 602
 603       nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 604                      clib_net_to_host_u32 (u->addr.as_u32));
 605       snat_ipfix_logging_max_entries_per_user (thread_index,
 606                                                sm->max_translations_per_user,
 607                                                u->addr.as_u32);
 608       return 0;
 609     }
 610
 611   oldest_index =
 612     clib_dlist_remove_head (tsm->list_pool,
 613                             u->sessions_per_user_list_head_index);
 614   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 615   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 616
 617   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 618   if (now >= sess_timeout_time)
 619     {
 620       // reuse old session
 621       clib_dlist_addtail (tsm->list_pool,
 622                           u->sessions_per_user_list_head_index, oldest_index);
 623       s = nat44_session_reuse_old (sm, u, s, thread_index, now);
 624     }
 625   else
 626     {
 627       // alloc new session
 628       clib_dlist_addhead (tsm->list_pool,
 629                           u->sessions_per_user_list_head_index, oldest_index);
 630     alloc_new:
 631       s = nat44_session_alloc_new (tsm, u, now);
 632       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 633                                pool_elts (tsm->sessions));
 634     }
 635   return s;
 636 }
 637
 638 void
 639 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 640                           int is_add)
 641 {
 642   fib_prefix_t prefix = {
 643     .fp_len = p_len,
 644     .fp_proto = FIB_PROTOCOL_IP4,
 645     .fp_addr = {
 646                 .ip4.as_u32 = addr->as_u32,
 647                 },
 648   };
 649   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 650
 651   if (is_add)
 652     fib_table_entry_update_one_path (fib_index,
 653                                      &prefix,
 654                                      nat_fib_src_low,
 655                                      (FIB_ENTRY_FLAG_CONNECTED |
 656                                       FIB_ENTRY_FLAG_LOCAL |
 657                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 658                                      DPO_PROTO_IP4,
 659                                      NULL,
 660                                      sw_if_index,
 661                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 662   else
 663     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 664 }
 665
 666 int
 667 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 668                   u8 twice_nat)
 669 {
 670   snat_address_t *ap;
 671   snat_interface_t *i;
 672   vlib_thread_main_t *tm = vlib_get_thread_main ();
 673
 674   if (twice_nat && !sm->endpoint_dependent)
 675     return VNET_API_ERROR_FEATURE_DISABLED;
 676
 677   /* Check if address already exists */
 678   /* *INDENT-OFF* */
 679   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 680     {
 681       if (ap->addr.as_u32 == addr->as_u32)
 682         return VNET_API_ERROR_VALUE_EXIST;
 683     }
 684   /* *INDENT-ON* */
 685
 686   if (twice_nat)
 687     vec_add2 (sm->twice_nat_addresses, ap, 1);
 688   else
 689     vec_add2 (sm->addresses, ap, 1);
 690
 691   ap->addr = *addr;
 692   if (vrf_id != ~0)
 693     ap->fib_index =
 694       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 695                                          nat_fib_src_low);
 696   else
 697     ap->fib_index = ~0;
 698 #define _(N, i, n, s) \
 699   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 700   ap->busy_##n##_ports = 0; \
 701   ap->busy_##n##_ports_per_thread = 0;\
 702   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 703   foreach_snat_protocol
 704 #undef _
 705     if (twice_nat)
 706     return 0;
 707
 708   /* Add external address to FIB */
 709   /* *INDENT-OFF* */
 710   pool_foreach (i, sm->interfaces,
 711   ({
 712     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 713       continue;
 714
 715     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 716     break;
 717   }));
 718   pool_foreach (i, sm->output_feature_interfaces,
 719   ({
 720     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 721       continue;
 722
 723     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 724     break;
 725   }));
 726   /* *INDENT-ON* */
 727
 728   return 0;
 729 }
 730
 731 static int
 732 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 733 {
 734   snat_static_mapping_t *m;
 735   /* *INDENT-OFF* */
 736   pool_foreach (m, sm->static_mappings,
 737   ({
 738       if (is_addr_only_static_mapping (m) ||
 739           is_out2in_only_static_mapping (m) ||
 740           is_identity_static_mapping (m))
 741         continue;
 742       if (m->external_addr.as_u32 == addr.as_u32)
 743         return 1;
 744   }));
 745   /* *INDENT-ON* */
 746
 747   return 0;
 748 }
 749
 750 static void
 751 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 752                                        ip4_address_t l_addr,
 753                                        u16 l_port,
 754                                        u32 sw_if_index,
 755                                        u16 e_port,
 756                                        u32 vrf_id,
 757                                        snat_protocol_t proto,
 758                                        int addr_only, int is_add, u8 * tag,
 759                                        int twice_nat, int out2in_only,
 760                                        int identity_nat)
 761 {
 762   snat_static_map_resolve_t *rp;
 763
 764   vec_add2 (sm->to_resolve, rp, 1);
 765   rp->l_addr.as_u32 = l_addr.as_u32;
 766   rp->l_port = l_port;
 767   rp->sw_if_index = sw_if_index;
 768   rp->e_port = e_port;
 769   rp->vrf_id = vrf_id;
 770   rp->proto = proto;
 771   rp->addr_only = addr_only;
 772   rp->is_add = is_add;
 773   rp->twice_nat = twice_nat;
 774   rp->out2in_only = out2in_only;
 775   rp->identity_nat = identity_nat;
 776   rp->tag = vec_dup (tag);
 777 }
 778
 779 static u32
 780 get_thread_idx_by_port (u16 e_port)
 781 {
 782   snat_main_t *sm = &snat_main;
 783   u32 thread_idx = sm->num_workers;
 784   if (sm->num_workers > 1)
 785     {
 786       thread_idx =
 787         sm->first_worker_index +
 788         sm->workers[(e_port - 1024) / sm->port_per_thread];
 789     }
 790   return thread_idx;
 791 }
 792
 793 int
 794 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 795                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 796                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 797                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 798                          u8 identity_nat)
 799 {
 800   snat_main_t *sm = &snat_main;
 801   snat_static_mapping_t *m;
 802   snat_session_key_t m_key;
 803   clib_bihash_kv_8_8_t kv, value;
 804   snat_address_t *a = 0;
 805   u32 fib_index = ~0;
 806   snat_interface_t *interface;
 807   int i;
 808   snat_main_per_thread_data_t *tsm;
 809   snat_user_key_t u_key;
 810   snat_user_t *u;
 811   dlist_elt_t *head, *elt;
 812   u32 elt_index, head_index;
 813   u32 ses_index;
 814   u64 user_index;
 815   snat_session_t *s;
 816   snat_static_map_resolve_t *rp, *rp_match = 0;
 817   nat44_lb_addr_port_t *local;
 818   u32 find = ~0;
 819
 820   if (!sm->endpoint_dependent)
 821     {
 822       if (twice_nat || out2in_only)
 823         return VNET_API_ERROR_FEATURE_DISABLED;
 824     }
 825
 826   /* If the external address is a specific interface address */
 827   if (sw_if_index != ~0)
 828     {
 829       ip4_address_t *first_int_addr;
 830
 831       for (i = 0; i < vec_len (sm->to_resolve); i++)
 832         {
 833           rp = sm->to_resolve + i;
 834           if (rp->sw_if_index != sw_if_index ||
 835               rp->l_addr.as_u32 != l_addr.as_u32 ||
 836               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 837             continue;
 838
 839           if (!addr_only)
 840             {
 841               if ((rp->l_port != l_port && rp->e_port != e_port)
 842                   || rp->proto != proto)
 843                 continue;
 844             }
 845
 846           rp_match = rp;
 847           break;
 848         }
 849
 850       /* Might be already set... */
 851       first_int_addr = ip4_interface_first_address
 852         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 853
 854       if (is_add)
 855         {
 856           if (rp_match)
 857             return VNET_API_ERROR_VALUE_EXIST;
 858
 859           snat_add_static_mapping_when_resolved
 860             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 861              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 862
 863           /* DHCP resolution required? */
 864           if (first_int_addr == 0)
 865             {
 866               return 0;
 867             }
 868           else
 869             {
 870               e_addr.as_u32 = first_int_addr->as_u32;
 871               /* Identity mapping? */
 872               if (l_addr.as_u32 == 0)
 873                 l_addr.as_u32 = e_addr.as_u32;
 874             }
 875         }
 876       else
 877         {
 878           if (!rp_match)
 879             return VNET_API_ERROR_NO_SUCH_ENTRY;
 880
 881           vec_del1 (sm->to_resolve, i);
 882
 883           if (first_int_addr)
 884             {
 885               e_addr.as_u32 = first_int_addr->as_u32;
 886               /* Identity mapping? */
 887               if (l_addr.as_u32 == 0)
 888                 l_addr.as_u32 = e_addr.as_u32;
 889             }
 890           else
 891             return 0;
 892         }
 893     }
 894
 895   m_key.addr = e_addr;
 896   m_key.port = addr_only ? 0 : e_port;
 897   m_key.protocol = addr_only ? 0 : proto;
 898   m_key.fib_index = 0;
 899   kv.key = m_key.as_u64;
 900   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 901     m = 0;
 902   else
 903     m = pool_elt_at_index (sm->static_mappings, value.value);
 904
 905   if (is_add)
 906     {
 907       if (m)
 908         {
 909           if (is_identity_static_mapping (m))
 910             {
 911               /* *INDENT-OFF* */
 912               pool_foreach (local, m->locals,
 913               ({
 914                 if (local->vrf_id == vrf_id)
 915                   return VNET_API_ERROR_VALUE_EXIST;
 916               }));
 917               /* *INDENT-ON* */
 918               pool_get (m->locals, local);
 919               local->vrf_id = vrf_id;
 920               local->fib_index =
 921                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 922                                                    nat_fib_src_low);
 923               m_key.addr = m->local_addr;
 924               m_key.port = m->local_port;
 925               m_key.protocol = m->proto;
 926               m_key.fib_index = local->fib_index;
 927               kv.key = m_key.as_u64;
 928               kv.value = m - sm->static_mappings;
 929               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 930               return 0;
 931             }
 932           else
 933             return VNET_API_ERROR_VALUE_EXIST;
 934         }
 935
 936       if (twice_nat && addr_only)
 937         return VNET_API_ERROR_UNSUPPORTED;
 938
 939       /* Convert VRF id to FIB index */
 940       if (vrf_id != ~0)
 941         fib_index =
 942           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 943                                              nat_fib_src_low);
 944       /* If not specified use inside VRF id from SNAT plugin startup config */
 945       else
 946         {
 947           fib_index = sm->inside_fib_index;
 948           vrf_id = sm->inside_vrf_id;
 949           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 950         }
 951
 952       if (!(out2in_only || identity_nat))
 953         {
 954           m_key.addr = l_addr;
 955           m_key.port = addr_only ? 0 : l_port;
 956           m_key.protocol = addr_only ? 0 : proto;
 957           m_key.fib_index = fib_index;
 958           kv.key = m_key.as_u64;
 959           if (!clib_bihash_search_8_8
 960               (&sm->static_mapping_by_local, &kv, &value))
 961             return VNET_API_ERROR_VALUE_EXIST;
 962         }
 963
 964       /* Find external address in allocated addresses and reserve port for
 965          address and port pair mapping when dynamic translations enabled */
 966       if (!(addr_only || sm->static_mapping_only || out2in_only))
 967         {
 968           for (i = 0; i < vec_len (sm->addresses); i++)
 969             {
 970               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 971                 {
 972                   a = sm->addresses + i;
 973                   /* External port must be unused */
 974                   switch (proto)
 975                     {
 976 #define _(N, j, n, s) \
 977                     case SNAT_PROTOCOL_##N: \
 978                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 979                         return VNET_API_ERROR_INVALID_VALUE; \
 980                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 981                       if (e_port > 1024) \
 982                         { \
 983                           a->busy_##n##_ports++; \
 984                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 985                         } \
 986                       break;
 987                       foreach_snat_protocol
 988 #undef _
 989                     default:
 990                       nat_elog_info ("unknown protocol");
 991                       return VNET_API_ERROR_INVALID_VALUE_2;
 992                     }
 993                   break;
 994                 }
 995             }
 996           /* External address must be allocated */
 997           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 998             {
 999               if (sw_if_index != ~0)
1000                 {
1001                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1002                     {
1003                       rp = sm->to_resolve + i;
1004                       if (rp->addr_only)
1005                         continue;
1006                       if (rp->sw_if_index != sw_if_index &&
1007                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1008                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1009                           rp->e_port != e_port && rp->proto != proto)
1010                         continue;
1011
1012                       vec_del1 (sm->to_resolve, i);
1013                       break;
1014                     }
1015                 }
1016               return VNET_API_ERROR_NO_SUCH_ENTRY;
1017             }
1018         }
1019
1020       pool_get (sm->static_mappings, m);
1021       clib_memset (m, 0, sizeof (*m));
1022       m->tag = vec_dup (tag);
1023       m->local_addr = l_addr;
1024       m->external_addr = e_addr;
1025       m->twice_nat = twice_nat;
1026       if (out2in_only)
1027         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1028       if (addr_only)
1029         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1030       if (identity_nat)
1031         {
1032           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1033           pool_get (m->locals, local);
1034           local->vrf_id = vrf_id;
1035           local->fib_index = fib_index;
1036         }
1037       else
1038         {
1039           m->vrf_id = vrf_id;
1040           m->fib_index = fib_index;
1041         }
1042       if (!addr_only)
1043         {
1044           m->local_port = l_port;
1045           m->external_port = e_port;
1046           m->proto = proto;
1047         }
1048
1049       if (sm->num_workers > 1)
1050         {
1051           ip4_header_t ip = {
1052             .src_address = m->local_addr,
1053           };
1054           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1055           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1056         }
1057       else
1058         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1059
1060       m_key.addr = m->local_addr;
1061       m_key.port = m->local_port;
1062       m_key.protocol = m->proto;
1063       m_key.fib_index = fib_index;
1064       kv.key = m_key.as_u64;
1065       kv.value = m - sm->static_mappings;
1066       if (!out2in_only)
1067         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1068
1069       m_key.addr = m->external_addr;
1070       m_key.port = m->external_port;
1071       m_key.fib_index = 0;
1072       kv.key = m_key.as_u64;
1073       kv.value = m - sm->static_mappings;
1074       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1075
1076       /* Delete dynamic sessions matching local address (+ local port) */
1077       if (!(sm->static_mapping_only))
1078         {
1079           u_key.addr = m->local_addr;
1080           u_key.fib_index = m->fib_index;
1081           kv.key = u_key.as_u64;
1082           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1083             {
1084               user_index = value.value;
1085               u = pool_elt_at_index (tsm->users, user_index);
1086               if (u->nsessions)
1087                 {
1088                   head_index = u->sessions_per_user_list_head_index;
1089                   head = pool_elt_at_index (tsm->list_pool, head_index);
1090                   elt_index = head->next;
1091                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1092                   ses_index = elt->value;
1093                   while (ses_index != ~0)
1094                     {
1095                       s = pool_elt_at_index (tsm->sessions, ses_index);
1096                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1097                       ses_index = elt->value;
1098
1099                       if (snat_is_session_static (s))
1100                         continue;
1101
1102                       if (!addr_only
1103                           && (clib_net_to_host_u16 (s->in2out.port) !=
1104                               m->local_port))
1105                         continue;
1106
1107                       nat_free_session_data (sm, s,
1108                                              tsm - sm->per_thread_data, 0);
1109                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1110
1111                       if (!addr_only && !sm->endpoint_dependent)
1112                         break;
1113                     }
1114                 }
1115             }
1116         }
1117     }
1118   else
1119     {
1120       if (!m)
1121         {
1122           if (sw_if_index != ~0)
1123             return 0;
1124           else
1125             return VNET_API_ERROR_NO_SUCH_ENTRY;
1126         }
1127
1128       if (identity_nat)
1129         {
1130           if (vrf_id == ~0)
1131             vrf_id = sm->inside_vrf_id;
1132
1133           /* *INDENT-OFF* */
1134           pool_foreach (local, m->locals,
1135           ({
1136             if (local->vrf_id == vrf_id)
1137               find = local - m->locals;
1138           }));
1139           /* *INDENT-ON* */
1140           if (find == ~0)
1141             return VNET_API_ERROR_NO_SUCH_ENTRY;
1142
1143           local = pool_elt_at_index (m->locals, find);
1144           fib_index = local->fib_index;
1145           pool_put (m->locals, local);
1146         }
1147       else
1148         fib_index = m->fib_index;
1149
1150       /* Free external address port */
1151       if (!(addr_only || sm->static_mapping_only || out2in_only))
1152         {
1153           for (i = 0; i < vec_len (sm->addresses); i++)
1154             {
1155               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1156                 {
1157                   a = sm->addresses + i;
1158                   switch (proto)
1159                     {
1160 #define _(N, j, n, s) \
1161                     case SNAT_PROTOCOL_##N: \
1162                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1163                       if (e_port > 1024) \
1164                         { \
1165                           a->busy_##n##_ports--; \
1166                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1167                         } \
1168                       break;
1169                       foreach_snat_protocol
1170 #undef _
1171                     default:
1172                       nat_elog_info ("unknown protocol");
1173                       return VNET_API_ERROR_INVALID_VALUE_2;
1174                     }
1175                   break;
1176                 }
1177             }
1178         }
1179
1180       if (sm->num_workers > 1)
1181         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1182       else
1183         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1184
1185       m_key.addr = m->local_addr;
1186       m_key.port = m->local_port;
1187       m_key.protocol = m->proto;
1188       m_key.fib_index = fib_index;
1189       kv.key = m_key.as_u64;
1190       if (!out2in_only)
1191         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1192
1193       /* Delete session(s) for static mapping if exist */
1194       if (!(sm->static_mapping_only) ||
1195           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1196         {
1197           u_key.addr = m->local_addr;
1198           u_key.fib_index = fib_index;
1199           kv.key = u_key.as_u64;
1200           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1201             {
1202               user_index = value.value;
1203               u = pool_elt_at_index (tsm->users, user_index);
1204               if (u->nstaticsessions)
1205                 {
1206                   head_index = u->sessions_per_user_list_head_index;
1207                   head = pool_elt_at_index (tsm->list_pool, head_index);
1208                   elt_index = head->next;
1209                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1210                   ses_index = elt->value;
1211                   while (ses_index != ~0)
1212                     {
1213                       s = pool_elt_at_index (tsm->sessions, ses_index);
1214                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1215                       ses_index = elt->value;
1216
1217                       if (!addr_only)
1218                         {
1219                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1220                               (clib_net_to_host_u16 (s->out2in.port) !=
1221                                e_port))
1222                             continue;
1223                         }
1224
1225                       if (is_lb_session (s))
1226                         continue;
1227
1228                       if (!snat_is_session_static (s))
1229                         continue;
1230
1231                       nat_free_session_data (sm, s,
1232                                              tsm - sm->per_thread_data, 0);
1233                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1234
1235                       if (!addr_only && !sm->endpoint_dependent)
1236                         break;
1237                     }
1238                 }
1239             }
1240         }
1241
1242       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1243       if (pool_elts (m->locals))
1244         return 0;
1245
1246       m_key.addr = m->external_addr;
1247       m_key.port = m->external_port;
1248       m_key.fib_index = 0;
1249       kv.key = m_key.as_u64;
1250       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1251
1252       vec_free (m->tag);
1253       vec_free (m->workers);
1254       /* Delete static mapping from pool */
1255       pool_put (sm->static_mappings, m);
1256     }
1257
1258   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1259     return 0;
1260
1261   /* Add/delete external address to FIB */
1262   /* *INDENT-OFF* */
1263   pool_foreach (interface, sm->interfaces,
1264   ({
1265     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1266       continue;
1267
1268     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1269     break;
1270   }));
1271   pool_foreach (interface, sm->output_feature_interfaces,
1272   ({
1273     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1274       continue;
1275
1276     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1277     break;
1278   }));
1279   /* *INDENT-ON* */
1280
1281   return 0;
1282 }
1283
1284 int
1285 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1286                                  snat_protocol_t proto,
1287                                  nat44_lb_addr_port_t * locals, u8 is_add,
1288                                  twice_nat_type_t twice_nat, u8 out2in_only,
1289                                  u8 * tag, u32 affinity)
1290 {
1291   snat_main_t *sm = &snat_main;
1292   snat_static_mapping_t *m;
1293   snat_session_key_t m_key;
1294   clib_bihash_kv_8_8_t kv, value;
1295   snat_address_t *a = 0;
1296   int i;
1297   nat44_lb_addr_port_t *local;
1298   u32 elt_index, head_index, ses_index;
1299   snat_main_per_thread_data_t *tsm;
1300   snat_user_key_t u_key;
1301   snat_user_t *u;
1302   snat_session_t *s;
1303   dlist_elt_t *head, *elt;
1304   uword *bitmap = 0;
1305
1306   if (!sm->endpoint_dependent)
1307     return VNET_API_ERROR_FEATURE_DISABLED;
1308
1309   m_key.addr = e_addr;
1310   m_key.port = e_port;
1311   m_key.protocol = proto;
1312   m_key.fib_index = 0;
1313   kv.key = m_key.as_u64;
1314   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1315     m = 0;
1316   else
1317     m = pool_elt_at_index (sm->static_mappings, value.value);
1318
1319   if (is_add)
1320     {
1321       if (m)
1322         return VNET_API_ERROR_VALUE_EXIST;
1323
1324       if (vec_len (locals) < 2)
1325         return VNET_API_ERROR_INVALID_VALUE;
1326
1327       /* Find external address in allocated addresses and reserve port for
1328          address and port pair mapping when dynamic translations enabled */
1329       if (!(sm->static_mapping_only || out2in_only))
1330         {
1331           for (i = 0; i < vec_len (sm->addresses); i++)
1332             {
1333               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1334                 {
1335                   a = sm->addresses + i;
1336                   /* External port must be unused */
1337                   switch (proto)
1338                     {
1339 #define _(N, j, n, s) \
1340                     case SNAT_PROTOCOL_##N: \
1341                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1342                         return VNET_API_ERROR_INVALID_VALUE; \
1343                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1344                       if (e_port > 1024) \
1345                         { \
1346                           a->busy_##n##_ports++; \
1347                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1348                         } \
1349                       break;
1350                       foreach_snat_protocol
1351 #undef _
1352                     default:
1353                       nat_elog_info ("unknown protocol");
1354                       return VNET_API_ERROR_INVALID_VALUE_2;
1355                     }
1356                   break;
1357                 }
1358             }
1359           /* External address must be allocated */
1360           if (!a)
1361             return VNET_API_ERROR_NO_SUCH_ENTRY;
1362         }
1363
1364       pool_get (sm->static_mappings, m);
1365       clib_memset (m, 0, sizeof (*m));
1366       m->tag = vec_dup (tag);
1367       m->external_addr = e_addr;
1368       m->external_port = e_port;
1369       m->proto = proto;
1370       m->twice_nat = twice_nat;
1371       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1372       if (out2in_only)
1373         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1374       m->affinity = affinity;
1375
1376       if (affinity)
1377         m->affinity_per_service_list_head_index =
1378           nat_affinity_get_per_service_list_head_index ();
1379       else
1380         m->affinity_per_service_list_head_index = ~0;
1381
1382       m_key.addr = m->external_addr;
1383       m_key.port = m->external_port;
1384       m_key.protocol = m->proto;
1385       m_key.fib_index = 0;
1386       kv.key = m_key.as_u64;
1387       kv.value = m - sm->static_mappings;
1388       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1389         {
1390           nat_elog_err ("static_mapping_by_external key add failed");
1391           return VNET_API_ERROR_UNSPECIFIED;
1392         }
1393
1394       m_key.fib_index = m->fib_index;
1395       for (i = 0; i < vec_len (locals); i++)
1396         {
1397           locals[i].fib_index =
1398             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1399                                                locals[i].vrf_id,
1400                                                nat_fib_src_low);
1401           m_key.addr = locals[i].addr;
1402           m_key.fib_index = locals[i].fib_index;
1403           if (!out2in_only)
1404             {
1405               m_key.port = locals[i].port;
1406               kv.key = m_key.as_u64;
1407               kv.value = m - sm->static_mappings;
1408               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1409             }
1410           locals[i].prefix = (i == 0) ? locals[i].probability :
1411             (locals[i - 1].prefix + locals[i].probability);
1412           pool_get (m->locals, local);
1413           *local = locals[i];
1414           if (sm->num_workers > 1)
1415             {
1416               ip4_header_t ip = {
1417                 .src_address = locals[i].addr,
1418               };
1419               bitmap =
1420                 clib_bitmap_set (bitmap,
1421                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1422                                  1);
1423             }
1424         }
1425
1426       /* Assign workers */
1427       if (sm->num_workers > 1)
1428         {
1429           /* *INDENT-OFF* */
1430           clib_bitmap_foreach (i, bitmap,
1431             ({
1432                vec_add1(m->workers, i);
1433             }));
1434           /* *INDENT-ON* */
1435         }
1436     }
1437   else
1438     {
1439       if (!m)
1440         return VNET_API_ERROR_NO_SUCH_ENTRY;
1441
1442       if (!is_lb_static_mapping (m))
1443         return VNET_API_ERROR_INVALID_VALUE;
1444
1445       /* Free external address port */
1446       if (!(sm->static_mapping_only || out2in_only))
1447         {
1448           for (i = 0; i < vec_len (sm->addresses); i++)
1449             {
1450               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1451                 {
1452                   a = sm->addresses + i;
1453                   switch (proto)
1454                     {
1455 #define _(N, j, n, s) \
1456                     case SNAT_PROTOCOL_##N: \
1457                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1458                       if (e_port > 1024) \
1459                         { \
1460                           a->busy_##n##_ports--; \
1461                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1462                         } \
1463                       break;
1464                       foreach_snat_protocol
1465 #undef _
1466                     default:
1467                       nat_elog_info ("unknown protocol");
1468                       return VNET_API_ERROR_INVALID_VALUE_2;
1469                     }
1470                   break;
1471                 }
1472             }
1473         }
1474
1475       m_key.addr = m->external_addr;
1476       m_key.port = m->external_port;
1477       m_key.protocol = m->proto;
1478       m_key.fib_index = 0;
1479       kv.key = m_key.as_u64;
1480       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1481         {
1482           nat_elog_err ("static_mapping_by_external key del failed");
1483           return VNET_API_ERROR_UNSPECIFIED;
1484         }
1485
1486       /* *INDENT-OFF* */
1487       pool_foreach (local, m->locals,
1488       ({
1489           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1490                             nat_fib_src_low);
1491           m_key.addr = local->addr;
1492           if (!out2in_only)
1493             {
1494               m_key.port = local->port;
1495               m_key.fib_index = local->fib_index;
1496               kv.key = m_key.as_u64;
1497               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1498                 {
1499                   nat_elog_err ("static_mapping_by_local key del failed");
1500                   return VNET_API_ERROR_UNSPECIFIED;
1501                 }
1502             }
1503
1504           if (sm->num_workers > 1)
1505             {
1506               ip4_header_t ip = {
1507                 .src_address = local->addr,
1508               };
1509               tsm = vec_elt_at_index (sm->per_thread_data,
1510                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1511             }
1512           else
1513             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1514
1515           /* Delete sessions */
1516           u_key.addr = local->addr;
1517           u_key.fib_index = local->fib_index;
1518           kv.key = u_key.as_u64;
1519           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1520             {
1521               u = pool_elt_at_index (tsm->users, value.value);
1522               if (u->nstaticsessions)
1523                 {
1524                   head_index = u->sessions_per_user_list_head_index;
1525                   head = pool_elt_at_index (tsm->list_pool, head_index);
1526                   elt_index = head->next;
1527                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1528                   ses_index = elt->value;
1529                   while (ses_index != ~0)
1530                     {
1531                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1532                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1533                       ses_index = elt->value;
1534
1535                       if (!(is_lb_session (s)))
1536                         continue;
1537
1538                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1539                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1540                         continue;
1541
1542                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1543                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1544                     }
1545                 }
1546             }
1547       }));
1548       /* *INDENT-ON* */
1549       if (m->affinity)
1550         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1551       pool_free (m->locals);
1552       vec_free (m->tag);
1553       vec_free (m->workers);
1554
1555       pool_put (sm->static_mappings, m);
1556     }
1557
1558   return 0;
1559 }
1560
1561 int
1562 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1563                                        ip4_address_t l_addr, u16 l_port,
1564                                        snat_protocol_t proto, u32 vrf_id,
1565                                        u8 probability, u8 is_add)
1566 {
1567   snat_main_t *sm = &snat_main;
1568   snat_static_mapping_t *m = 0;
1569   snat_session_key_t m_key;
1570   clib_bihash_kv_8_8_t kv, value;
1571   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1572   snat_main_per_thread_data_t *tsm;
1573   snat_user_key_t u_key;
1574   snat_user_t *u;
1575   snat_session_t *s;
1576   dlist_elt_t *head, *elt;
1577   u32 elt_index, head_index, ses_index, *locals = 0;
1578   uword *bitmap = 0;
1579   int i;
1580
1581   if (!sm->endpoint_dependent)
1582     return VNET_API_ERROR_FEATURE_DISABLED;
1583
1584   m_key.addr = e_addr;
1585   m_key.port = e_port;
1586   m_key.protocol = proto;
1587   m_key.fib_index = 0;
1588   kv.key = m_key.as_u64;
1589   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1590     m = pool_elt_at_index (sm->static_mappings, value.value);
1591
1592   if (!m)
1593     return VNET_API_ERROR_NO_SUCH_ENTRY;
1594
1595   if (!is_lb_static_mapping (m))
1596     return VNET_API_ERROR_INVALID_VALUE;
1597
1598   /* *INDENT-OFF* */
1599   pool_foreach (local, m->locals,
1600   ({
1601     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1602         (local->vrf_id == vrf_id))
1603       {
1604         match_local = local;
1605         break;
1606       }
1607   }));
1608   /* *INDENT-ON* */
1609
1610   if (is_add)
1611     {
1612       if (match_local)
1613         return VNET_API_ERROR_VALUE_EXIST;
1614
1615       pool_get (m->locals, local);
1616       clib_memset (local, 0, sizeof (*local));
1617       local->addr.as_u32 = l_addr.as_u32;
1618       local->port = l_port;
1619       local->probability = probability;
1620       local->vrf_id = vrf_id;
1621       local->fib_index =
1622         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1623                                            nat_fib_src_low);
1624
1625       if (!is_out2in_only_static_mapping (m))
1626         {
1627           m_key.addr = l_addr;
1628           m_key.port = l_port;
1629           m_key.fib_index = local->fib_index;
1630           kv.key = m_key.as_u64;
1631           kv.value = m - sm->static_mappings;
1632           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1633             nat_elog_err ("static_mapping_by_local key add failed");
1634         }
1635     }
1636   else
1637     {
1638       if (!match_local)
1639         return VNET_API_ERROR_NO_SUCH_ENTRY;
1640
1641       if (pool_elts (m->locals) < 3)
1642         return VNET_API_ERROR_UNSPECIFIED;
1643
1644       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1645                         nat_fib_src_low);
1646
1647       if (!is_out2in_only_static_mapping (m))
1648         {
1649           m_key.addr = l_addr;
1650           m_key.port = l_port;
1651           m_key.fib_index = match_local->fib_index;
1652           kv.key = m_key.as_u64;
1653           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1654             nat_elog_err ("static_mapping_by_local key del failed");
1655         }
1656
1657       if (sm->num_workers > 1)
1658         {
1659           ip4_header_t ip = {
1660             .src_address = local->addr,
1661           };
1662           tsm = vec_elt_at_index (sm->per_thread_data,
1663                                   sm->worker_in2out_cb (&ip, m->fib_index,
1664                                                         0));
1665         }
1666       else
1667         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1668
1669       /* Delete sessions */
1670       u_key.addr = match_local->addr;
1671       u_key.fib_index = match_local->fib_index;
1672       kv.key = u_key.as_u64;
1673       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1674         {
1675           u = pool_elt_at_index (tsm->users, value.value);
1676           if (u->nstaticsessions)
1677             {
1678               head_index = u->sessions_per_user_list_head_index;
1679               head = pool_elt_at_index (tsm->list_pool, head_index);
1680               elt_index = head->next;
1681               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1682               ses_index = elt->value;
1683               while (ses_index != ~0)
1684                 {
1685                   s = pool_elt_at_index (tsm->sessions, ses_index);
1686                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1687                   ses_index = elt->value;
1688
1689                   if (!(is_lb_session (s)))
1690                     continue;
1691
1692                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1693                       (clib_net_to_host_u16 (s->in2out.port) !=
1694                        match_local->port))
1695                     continue;
1696
1697                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1698                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1699                 }
1700             }
1701         }
1702
1703       pool_put (m->locals, match_local);
1704     }
1705
1706   vec_free (m->workers);
1707
1708   /* *INDENT-OFF* */
1709   pool_foreach (local, m->locals,
1710   ({
1711     vec_add1 (locals, local - m->locals);
1712     if (sm->num_workers > 1)
1713       {
1714         ip4_header_t ip;
1715         ip.src_address.as_u32 = local->addr.as_u32,
1716         bitmap = clib_bitmap_set (bitmap,
1717                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1718                                   1);
1719       }
1720   }));
1721   /* *INDENT-ON* */
1722
1723   ASSERT (vec_len (locals) > 1);
1724
1725   local = pool_elt_at_index (m->locals, locals[0]);
1726   local->prefix = local->probability;
1727   for (i = 1; i < vec_len (locals); i++)
1728     {
1729       local = pool_elt_at_index (m->locals, locals[i]);
1730       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1731       local->prefix = local->probability + prev_local->prefix;
1732     }
1733
1734   /* Assign workers */
1735   if (sm->num_workers > 1)
1736     {
1737       /* *INDENT-OFF* */
1738       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1739       /* *INDENT-ON* */
1740     }
1741
1742   return 0;
1743 }
1744
1745 int
1746 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1747                   u8 twice_nat)
1748 {
1749   snat_address_t *a = 0;
1750   snat_session_t *ses;
1751   u32 *ses_to_be_removed = 0, *ses_index;
1752   snat_main_per_thread_data_t *tsm;
1753   snat_static_mapping_t *m;
1754   snat_interface_t *interface;
1755   int i;
1756   snat_address_t *addresses =
1757     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1758
1759   /* Find SNAT address */
1760   for (i = 0; i < vec_len (addresses); i++)
1761     {
1762       if (addresses[i].addr.as_u32 == addr.as_u32)
1763         {
1764           a = addresses + i;
1765           break;
1766         }
1767     }
1768   if (!a)
1769     return VNET_API_ERROR_NO_SUCH_ENTRY;
1770
1771   if (delete_sm)
1772     {
1773       /* *INDENT-OFF* */
1774       pool_foreach (m, sm->static_mappings,
1775       ({
1776           if (m->external_addr.as_u32 == addr.as_u32)
1777             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1778                                             m->local_port, m->external_port,
1779                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1780                                             m->proto, 0, m->twice_nat,
1781                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1782       }));
1783       /* *INDENT-ON* */
1784     }
1785   else
1786     {
1787       /* Check if address is used in some static mapping */
1788       if (is_snat_address_used_in_static_mapping (sm, addr))
1789         {
1790           nat_elog_notice ("address used in static mapping");
1791           return VNET_API_ERROR_UNSPECIFIED;
1792         }
1793     }
1794
1795   if (a->fib_index != ~0)
1796     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1797
1798   /* Delete sessions using address */
1799   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1800     {
1801       /* *INDENT-OFF* */
1802       vec_foreach (tsm, sm->per_thread_data)
1803         {
1804           pool_foreach (ses, tsm->sessions, ({
1805             if (ses->out2in.addr.as_u32 == addr.as_u32)
1806               {
1807                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1808                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1809               }
1810           }));
1811
1812           vec_foreach (ses_index, ses_to_be_removed)
1813             {
1814               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1815               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1816             }
1817
1818           vec_free (ses_to_be_removed);
1819         }
1820       /* *INDENT-ON* */
1821     }
1822
1823 #define _(N, i, n, s) \
1824   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1825   vec_free (a->busy_##n##_ports_per_thread);
1826   foreach_snat_protocol
1827 #undef _
1828     if (twice_nat)
1829     {
1830       vec_del1 (sm->twice_nat_addresses, i);
1831       return 0;
1832     }
1833   else
1834     vec_del1 (sm->addresses, i);
1835
1836   /* Delete external address from FIB */
1837   /* *INDENT-OFF* */
1838   pool_foreach (interface, sm->interfaces,
1839   ({
1840     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1841       continue;
1842
1843     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1844     break;
1845   }));
1846   pool_foreach (interface, sm->output_feature_interfaces,
1847   ({
1848     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1849       continue;
1850
1851     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1852     break;
1853   }));
1854   /* *INDENT-ON* */
1855
1856   return 0;
1857 }
1858
1859 int
1860 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1861 {
1862   snat_main_t *sm = &snat_main;
1863   snat_interface_t *i;
1864   const char *feature_name, *del_feature_name;
1865   snat_address_t *ap;
1866   snat_static_mapping_t *m;
1867   snat_det_map_t *dm;
1868   nat_outside_fib_t *outside_fib;
1869   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1870                                                        sw_if_index);
1871
1872   if (sm->out2in_dpo && !is_inside)
1873     return VNET_API_ERROR_UNSUPPORTED;
1874
1875   /* *INDENT-OFF* */
1876   pool_foreach (i, sm->output_feature_interfaces,
1877   ({
1878     if (i->sw_if_index == sw_if_index)
1879       return VNET_API_ERROR_VALUE_EXIST;
1880   }));
1881   /* *INDENT-ON* */
1882
1883   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1884     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1885   else
1886     {
1887       if (sm->num_workers > 1 && !sm->deterministic)
1888         feature_name =
1889           is_inside ? "nat44-in2out-worker-handoff" :
1890           "nat44-out2in-worker-handoff";
1891       else if (sm->deterministic)
1892         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1893       else if (sm->endpoint_dependent)
1894         {
1895           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1896         }
1897       else
1898         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1899     }
1900
1901   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1902     sm->fq_in2out_index =
1903       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1904
1905   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1906     sm->fq_out2in_index =
1907       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1908
1909   if (!is_inside)
1910     {
1911       /* *INDENT-OFF* */
1912       vec_foreach (outside_fib, sm->outside_fibs)
1913         {
1914           if (outside_fib->fib_index == fib_index)
1915             {
1916               if (is_del)
1917                 {
1918                   outside_fib->refcount--;
1919                   if (!outside_fib->refcount)
1920                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1921                 }
1922               else
1923                 outside_fib->refcount++;
1924               goto feature_set;
1925             }
1926         }
1927       /* *INDENT-ON* */
1928       if (!is_del)
1929         {
1930           vec_add2 (sm->outside_fibs, outside_fib, 1);
1931           outside_fib->refcount = 1;
1932           outside_fib->fib_index = fib_index;
1933         }
1934     }
1935 feature_set:
1936   /* *INDENT-OFF* */
1937   pool_foreach (i, sm->interfaces,
1938   ({
1939     if (i->sw_if_index == sw_if_index)
1940       {
1941         if (is_del)
1942           {
1943             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1944               {
1945                 if (is_inside)
1946                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1947                 else
1948                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1949
1950                 if (sm->num_workers > 1 && !sm->deterministic)
1951                   {
1952                     del_feature_name = "nat44-handoff-classify";
1953                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1954                                                  "nat44-out2in-worker-handoff";
1955                   }
1956                 else if (sm->deterministic)
1957                   {
1958                     del_feature_name = "nat44-det-classify";
1959                     feature_name = !is_inside ?  "nat44-det-in2out" :
1960                                                  "nat44-det-out2in";
1961                   }
1962                 else if (sm->endpoint_dependent)
1963                   {
1964                     del_feature_name = "nat44-ed-classify";
1965                     feature_name = !is_inside ?  "nat-pre-in2out" :
1966                                                  "nat-pre-out2in";
1967                   }
1968                 else
1969                   {
1970                     del_feature_name = "nat44-classify";
1971                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1972                   }
1973
1974                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1975                 if (rv)
1976                   return rv;
1977                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1978                                              sw_if_index, 0, 0, 0);
1979                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1980                                              sw_if_index, 1, 0, 0);
1981                 if (!is_inside)
1982                   {
1983                     if (sm->endpoint_dependent)
1984                       vnet_feature_enable_disable ("ip4-local",
1985                                                    "nat44-ed-hairpinning",
1986                                                    sw_if_index, 1, 0, 0);
1987                     else if (!sm->deterministic)
1988                       vnet_feature_enable_disable ("ip4-local",
1989                                                    "nat44-hairpinning",
1990                                                    sw_if_index, 1, 0, 0);
1991                   }
1992               }
1993             else
1994               {
1995                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1996                 if (rv)
1997                   return rv;
1998                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1999                                              sw_if_index, 0, 0, 0);
2000                 pool_put (sm->interfaces, i);
2001                 if (is_inside)
2002                   {
2003                     if (sm->endpoint_dependent)
2004                       vnet_feature_enable_disable ("ip4-local",
2005                                                    "nat44-ed-hairpinning",
2006                                                    sw_if_index, 0, 0, 0);
2007                     else if (!sm->deterministic)
2008                       vnet_feature_enable_disable ("ip4-local",
2009                                                    "nat44-hairpinning",
2010                                                    sw_if_index, 0, 0, 0);
2011                   }
2012               }
2013           }
2014         else
2015           {
2016             if ((nat_interface_is_inside(i) && is_inside) ||
2017                 (nat_interface_is_outside(i) && !is_inside))
2018               return 0;
2019
2020             if (sm->num_workers > 1 && !sm->deterministic)
2021               {
2022                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2023                                                  "nat44-out2in-worker-handoff";
2024                 feature_name = "nat44-handoff-classify";
2025               }
2026             else if (sm->deterministic)
2027               {
2028                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
2029                                                  "nat44-det-out2in";
2030                 feature_name = "nat44-det-classify";
2031               }
2032             else if (sm->endpoint_dependent)
2033               {
2034                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2035                                                  "nat-pre-out2in";
2036
2037                 feature_name = "nat44-ed-classify";
2038               }
2039             else
2040               {
2041                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2042                 feature_name = "nat44-classify";
2043               }
2044
2045             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2046             if (rv)
2047               return rv;
2048             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2049                                          sw_if_index, 0, 0, 0);
2050             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2051                                          sw_if_index, 1, 0, 0);
2052             if (!is_inside)
2053               {
2054                 if (sm->endpoint_dependent)
2055                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2056                                                sw_if_index, 0, 0, 0);
2057                 else if (!sm->deterministic)
2058                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2059                                                sw_if_index, 0, 0, 0);
2060               }
2061             goto set_flags;
2062           }
2063
2064         goto fib;
2065       }
2066   }));
2067   /* *INDENT-ON* */
2068
2069   if (is_del)
2070     return VNET_API_ERROR_NO_SUCH_ENTRY;
2071
2072   pool_get (sm->interfaces, i);
2073   i->sw_if_index = sw_if_index;
2074   i->flags = 0;
2075   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2076                                0);
2077
2078   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2079   if (rv)
2080     return rv;
2081
2082   if (is_inside && !sm->out2in_dpo)
2083     {
2084       if (sm->endpoint_dependent)
2085         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2086                                      sw_if_index, 1, 0, 0);
2087       else if (!sm->deterministic)
2088         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2089                                      sw_if_index, 1, 0, 0);
2090     }
2091
2092 set_flags:
2093   if (is_inside)
2094     {
2095       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2096       return 0;
2097     }
2098   else
2099     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2100
2101   /* Add/delete external addresses to FIB */
2102 fib:
2103   /* *INDENT-OFF* */
2104   vec_foreach (ap, sm->addresses)
2105     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2106
2107   pool_foreach (m, sm->static_mappings,
2108   ({
2109     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2110       continue;
2111
2112     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2113   }));
2114
2115   pool_foreach (dm, sm->det_maps,
2116   ({
2117     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2118   }));
2119   /* *INDENT-ON* */
2120
2121   return 0;
2122 }
2123
2124 int
2125 snat_interface_add_del_output_feature (u32 sw_if_index,
2126                                        u8 is_inside, int is_del)
2127 {
2128   snat_main_t *sm = &snat_main;
2129   snat_interface_t *i;
2130   snat_address_t *ap;
2131   snat_static_mapping_t *m;
2132   nat_outside_fib_t *outside_fib;
2133   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2134                                                        sw_if_index);
2135
2136
2137   if (sm->deterministic ||
2138       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2139     return VNET_API_ERROR_UNSUPPORTED;
2140
2141   /* *INDENT-OFF* */
2142   pool_foreach (i, sm->interfaces,
2143   ({
2144     if (i->sw_if_index == sw_if_index)
2145       return VNET_API_ERROR_VALUE_EXIST;
2146   }));
2147   /* *INDENT-ON* */
2148
2149   if (!is_inside)
2150     {
2151       /* *INDENT-OFF* */
2152       vec_foreach (outside_fib, sm->outside_fibs)
2153         {
2154           if (outside_fib->fib_index == fib_index)
2155             {
2156               if (is_del)
2157                 {
2158                   outside_fib->refcount--;
2159                   if (!outside_fib->refcount)
2160                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2161                 }
2162               else
2163                 outside_fib->refcount++;
2164               goto feature_set;
2165             }
2166         }
2167       /* *INDENT-ON* */
2168       if (!is_del)
2169         {
2170           vec_add2 (sm->outside_fibs, outside_fib, 1);
2171           outside_fib->refcount = 1;
2172           outside_fib->fib_index = fib_index;
2173         }
2174     }
2175
2176 feature_set:
2177   if (is_inside)
2178     {
2179       if (sm->endpoint_dependent)
2180         {
2181           int rv =
2182             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2183           if (rv)
2184             return rv;
2185           rv =
2186             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2187                                                             !is_del);
2188           if (rv)
2189             return rv;
2190           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2191                                        sw_if_index, !is_del, 0, 0);
2192           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2193                                        sw_if_index, !is_del, 0, 0);
2194         }
2195       else
2196         {
2197           int rv =
2198             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2199           if (rv)
2200             return rv;
2201           rv =
2202             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2203                                                             !is_del);
2204           if (rv)
2205             return rv;
2206           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2207                                        sw_if_index, !is_del, 0, 0);
2208           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2209                                        sw_if_index, !is_del, 0, 0);
2210         }
2211       goto fq;
2212     }
2213
2214   if (sm->num_workers > 1)
2215     {
2216       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2217       if (rv)
2218         return rv;
2219       rv =
2220         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2221       if (rv)
2222         return rv;
2223       vnet_feature_enable_disable ("ip4-unicast",
2224                                    "nat44-out2in-worker-handoff",
2225                                    sw_if_index, !is_del, 0, 0);
2226       vnet_feature_enable_disable ("ip4-output",
2227                                    "nat44-in2out-output-worker-handoff",
2228                                    sw_if_index, !is_del, 0, 0);
2229     }
2230   else
2231     {
2232       if (sm->endpoint_dependent)
2233         {
2234           int rv =
2235             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2236           if (rv)
2237             return rv;
2238           rv =
2239             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2240                                                             !is_del);
2241           if (rv)
2242             return rv;
2243           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2244                                        sw_if_index, !is_del, 0, 0);
2245           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2246                                        sw_if_index, !is_del, 0, 0);
2247         }
2248       else
2249         {
2250           int rv =
2251             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2252           if (rv)
2253             return rv;
2254           rv =
2255             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2256                                                             !is_del);
2257           if (rv)
2258             return rv;
2259           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2260                                        sw_if_index, !is_del, 0, 0);
2261           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2262                                        sw_if_index, !is_del, 0, 0);
2263         }
2264     }
2265
2266 fq:
2267   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2268     sm->fq_in2out_output_index =
2269       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2270
2271   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2272     sm->fq_out2in_index =
2273       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2274
2275   /* *INDENT-OFF* */
2276   pool_foreach (i, sm->output_feature_interfaces,
2277   ({
2278     if (i->sw_if_index == sw_if_index)
2279       {
2280         if (is_del)
2281           pool_put (sm->output_feature_interfaces, i);
2282         else
2283           return VNET_API_ERROR_VALUE_EXIST;
2284
2285         goto fib;
2286       }
2287   }));
2288   /* *INDENT-ON* */
2289
2290   if (is_del)
2291     return VNET_API_ERROR_NO_SUCH_ENTRY;
2292
2293   pool_get (sm->output_feature_interfaces, i);
2294   i->sw_if_index = sw_if_index;
2295   i->flags = 0;
2296   if (is_inside)
2297     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2298   else
2299     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2300
2301   /* Add/delete external addresses to FIB */
2302 fib:
2303   if (is_inside)
2304     return 0;
2305
2306   /* *INDENT-OFF* */
2307   vec_foreach (ap, sm->addresses)
2308     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2309
2310   pool_foreach (m, sm->static_mappings,
2311   ({
2312     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2313       continue;
2314
2315     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2316   }));
2317   /* *INDENT-ON* */
2318
2319   return 0;
2320 }
2321
2322 int
2323 snat_set_workers (uword * bitmap)
2324 {
2325   snat_main_t *sm = &snat_main;
2326   int i, j = 0;
2327
2328   if (sm->num_workers < 2)
2329     return VNET_API_ERROR_FEATURE_DISABLED;
2330
2331   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2332     return VNET_API_ERROR_INVALID_WORKER;
2333
2334   vec_free (sm->workers);
2335   /* *INDENT-OFF* */
2336   clib_bitmap_foreach (i, bitmap,
2337     ({
2338       vec_add1(sm->workers, i);
2339       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2340       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2341       j++;
2342     }));
2343   /* *INDENT-ON* */
2344
2345   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2346   sm->num_snat_thread = _vec_len (sm->workers);
2347
2348   return 0;
2349 }
2350
2351 static void
2352 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2353                          u32 old_fib_index)
2354 {
2355   snat_main_t *sm = &snat_main;
2356   nat_outside_fib_t *outside_fib;
2357   snat_interface_t *i;
2358   u8 is_add = 1;
2359   u8 match = 0;
2360
2361   if (new_fib_index == old_fib_index)
2362     return;
2363
2364   if (!vec_len (sm->outside_fibs))
2365     return;
2366
2367   /* *INDENT-OFF* */
2368   pool_foreach (i, sm->interfaces,
2369     ({
2370       if (i->sw_if_index == sw_if_index)
2371         {
2372           if (!(nat_interface_is_outside (i)))
2373             return;
2374           match = 1;
2375         }
2376     }));
2377
2378   pool_foreach (i, sm->output_feature_interfaces,
2379     ({
2380       if (i->sw_if_index == sw_if_index)
2381         {
2382           if (!(nat_interface_is_outside (i)))
2383             return;
2384           match = 1;
2385         }
2386     }));
2387   /* *INDENT-ON* */
2388
2389   if (!match)
2390     return;
2391
2392   vec_foreach (outside_fib, sm->outside_fibs)
2393   {
2394     if (outside_fib->fib_index == old_fib_index)
2395       {
2396         outside_fib->refcount--;
2397         if (!outside_fib->refcount)
2398           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2399         break;
2400       }
2401   }
2402
2403   vec_foreach (outside_fib, sm->outside_fibs)
2404   {
2405     if (outside_fib->fib_index == new_fib_index)
2406       {
2407         outside_fib->refcount++;
2408         is_add = 0;
2409         break;
2410       }
2411   }
2412
2413   if (is_add)
2414     {
2415       vec_add2 (sm->outside_fibs, outside_fib, 1);
2416       outside_fib->refcount = 1;
2417       outside_fib->fib_index = new_fib_index;
2418     }
2419 }
2420
2421 static void
2422 snat_ip4_table_bind (ip4_main_t * im,
2423                      uword opaque,
2424                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2425 {
2426   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2427 }
2428
2429 static void
2430 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2431                                        uword opaque,
2432                                        u32 sw_if_index,
2433                                        ip4_address_t * address,
2434                                        u32 address_length,
2435                                        u32 if_address_index, u32 is_delete);
2436
2437 static void
2438 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2439                                  uword opaque,
2440                                  u32 sw_if_index,
2441                                  ip4_address_t * address,
2442                                  u32 address_length,
2443                                  u32 if_address_index, u32 is_delete);
2444
2445 static int
2446 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2447                                  u32 fib_index,
2448                                  u32 thread_index,
2449                                  snat_session_key_t * k,
2450                                  u16 port_per_thread, u32 snat_thread_index);
2451
2452 static clib_error_t *
2453 snat_init (vlib_main_t * vm)
2454 {
2455   snat_main_t *sm = &snat_main;
2456   clib_error_t *error = 0;
2457   ip4_main_t *im = &ip4_main;
2458   ip_lookup_main_t *lm = &im->lookup_main;
2459   uword *p;
2460   vlib_thread_registration_t *tr;
2461   vlib_thread_main_t *tm = vlib_get_thread_main ();
2462   uword *bitmap = 0;
2463   u32 i;
2464   ip4_add_del_interface_address_callback_t cb4;
2465   vlib_node_t *node;
2466
2467   sm->vlib_main = vm;
2468   sm->vnet_main = vnet_get_main ();
2469   sm->ip4_main = im;
2470   sm->ip4_lookup_main = lm;
2471   sm->api_main = vlibapi_get_main ();
2472   sm->first_worker_index = 0;
2473   sm->num_workers = 0;
2474   sm->num_snat_thread = 1;
2475   sm->workers = 0;
2476   sm->port_per_thread = 0xffff - 1024;
2477   sm->fq_in2out_index = ~0;
2478   sm->fq_in2out_output_index = ~0;
2479   sm->fq_out2in_index = ~0;
2480
2481   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2482   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2483   sm->forwarding_enabled = 0;
2484   sm->log_class = vlib_log_register_class ("nat", 0);
2485   sm->log_level = SNAT_LOG_ERROR;
2486   sm->mss_clamping = 0;
2487
2488   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2489   sm->error_node_index = node->index;
2490
2491   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2492   sm->pre_in2out_node_index = node->index;
2493   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2494   sm->pre_out2in_node_index = node->index;
2495
2496   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2497   sm->pre_in2out_node_index = node->index;
2498
2499   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2500   sm->pre_out2in_node_index = node->index;
2501
2502   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2503   sm->in2out_node_index = node->index;
2504   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2505   sm->in2out_output_node_index = node->index;
2506   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2507   sm->in2out_fast_node_index = node->index;
2508   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2509   sm->in2out_slowpath_node_index = node->index;
2510   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2511   sm->in2out_slowpath_output_node_index = node->index;
2512
2513   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2514   sm->ed_in2out_node_index = node->index;
2515   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2516   sm->ed_in2out_slowpath_node_index = node->index;
2517
2518   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2519   sm->out2in_node_index = node->index;
2520   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2521   sm->out2in_fast_node_index = node->index;
2522
2523   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2524   sm->ed_out2in_node_index = node->index;
2525   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2526   sm->ed_out2in_slowpath_node_index = node->index;
2527
2528   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2529   sm->det_in2out_node_index = node->index;
2530   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2531   sm->det_out2in_node_index = node->index;
2532
2533   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2534   sm->hairpinning_node_index = node->index;
2535   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2536   sm->hairpin_dst_node_index = node->index;
2537   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2538   sm->hairpin_src_node_index = node->index;
2539   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2540   sm->ed_hairpinning_node_index = node->index;
2541   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2542   sm->ed_hairpin_dst_node_index = node->index;
2543   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2544   sm->ed_hairpin_src_node_index = node->index;
2545
2546   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2547   if (p)
2548     {
2549       tr = (vlib_thread_registration_t *) p[0];
2550       if (tr)
2551         {
2552           sm->num_workers = tr->count;
2553           sm->first_worker_index = tr->first_index;
2554         }
2555     }
2556
2557   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2558
2559   /* Use all available workers by default */
2560   if (sm->num_workers > 1)
2561     {
2562       for (i = 0; i < sm->num_workers; i++)
2563         bitmap = clib_bitmap_set (bitmap, i, 1);
2564       snat_set_workers (bitmap);
2565       clib_bitmap_free (bitmap);
2566     }
2567   else
2568     {
2569       sm->per_thread_data[0].snat_thread_index = 0;
2570     }
2571
2572   error = snat_api_init (vm, sm);
2573   if (error)
2574     return error;
2575
2576   /* Set up the interface address add/del callback */
2577   cb4.function = snat_ip4_add_del_interface_address_cb;
2578   cb4.function_opaque = 0;
2579
2580   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2581
2582   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2583   cb4.function_opaque = 0;
2584
2585   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2586
2587   nat_dpo_module_init ();
2588
2589   /* Init counters */
2590   sm->total_users.name = "total-users";
2591   sm->total_users.stat_segment_name = "/nat44/total-users";
2592   vlib_validate_simple_counter (&sm->total_users, 0);
2593   vlib_zero_simple_counter (&sm->total_users, 0);
2594   sm->total_sessions.name = "total-sessions";
2595   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2596   vlib_validate_simple_counter (&sm->total_sessions, 0);
2597   vlib_zero_simple_counter (&sm->total_sessions, 0);
2598
2599   /* Init IPFIX logging */
2600   snat_ipfix_logging_init (vm);
2601
2602   /* Init NAT64 */
2603   error = nat64_init (vm);
2604   if (error)
2605     return error;
2606
2607   nat66_init (vm);
2608
2609   ip4_table_bind_callback_t cbt4 = {
2610     .function = snat_ip4_table_bind,
2611   };
2612   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2613
2614   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2615                                         FIB_SOURCE_PRIORITY_HI,
2616                                         FIB_SOURCE_BH_SIMPLE);
2617   nat_fib_src_low = fib_source_allocate ("nat-low",
2618                                          FIB_SOURCE_PRIORITY_LOW,
2619                                          FIB_SOURCE_BH_SIMPLE);
2620
2621   return error;
2622 }
2623
2624 VLIB_INIT_FUNCTION (snat_init);
2625
2626 void
2627 snat_free_outside_address_and_port (snat_address_t * addresses,
2628                                     u32 thread_index, snat_session_key_t * k)
2629 {
2630   snat_address_t *a;
2631   u32 address_index;
2632   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2633
2634   for (address_index = 0; address_index < vec_len (addresses);
2635        address_index++)
2636     {
2637       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2638         break;
2639     }
2640
2641   ASSERT (address_index < vec_len (addresses));
2642
2643   a = addresses + address_index;
2644
2645   switch (k->protocol)
2646     {
2647 #define _(N, i, n, s) \
2648     case SNAT_PROTOCOL_##N: \
2649       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2650         port_host_byte_order) == 1); \
2651       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2652         port_host_byte_order, 0); \
2653       a->busy_##n##_ports--; \
2654       a->busy_##n##_ports_per_thread[thread_index]--; \
2655       break;
2656       foreach_snat_protocol
2657 #undef _
2658     default:
2659       nat_elog_info ("unknown protocol");
2660       return;
2661     }
2662 }
2663
2664 static int
2665 nat_set_outside_address_and_port (snat_address_t * addresses,
2666                                   u32 thread_index, snat_session_key_t * k)
2667 {
2668   snat_address_t *a = 0;
2669   u32 address_index;
2670   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2671
2672   for (address_index = 0; address_index < vec_len (addresses);
2673        address_index++)
2674     {
2675       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2676         continue;
2677
2678       a = addresses + address_index;
2679       switch (k->protocol)
2680         {
2681 #define _(N, j, n, s) \
2682         case SNAT_PROTOCOL_##N: \
2683           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2684             return VNET_API_ERROR_INSTANCE_IN_USE; \
2685           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2686           a->busy_##n##_ports_per_thread[thread_index]++; \
2687           a->busy_##n##_ports++; \
2688           return 0;
2689           foreach_snat_protocol
2690 #undef _
2691         default:
2692           nat_elog_info ("unknown protocol");
2693           return 1;
2694         }
2695     }
2696
2697   return VNET_API_ERROR_NO_SUCH_ENTRY;
2698 }
2699
2700 int
2701 snat_static_mapping_match (snat_main_t * sm,
2702                            snat_session_key_t match,
2703                            snat_session_key_t * mapping,
2704                            u8 by_external,
2705                            u8 * is_addr_only,
2706                            twice_nat_type_t * twice_nat,
2707                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2708                            u8 * is_identity_nat)
2709 {
2710   clib_bihash_kv_8_8_t kv, value;
2711   snat_static_mapping_t *m;
2712   snat_session_key_t m_key;
2713   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2714   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2715   u8 backend_index;
2716   nat44_lb_addr_port_t *local;
2717
2718   m_key.fib_index = match.fib_index;
2719   if (by_external)
2720     {
2721       mapping_hash = &sm->static_mapping_by_external;
2722       m_key.fib_index = 0;
2723     }
2724
2725   m_key.addr = match.addr;
2726   m_key.port = clib_net_to_host_u16 (match.port);
2727   m_key.protocol = match.protocol;
2728
2729   kv.key = m_key.as_u64;
2730
2731   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2732     {
2733       /* Try address only mapping */
2734       m_key.port = 0;
2735       m_key.protocol = 0;
2736       kv.key = m_key.as_u64;
2737       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2738         return 1;
2739     }
2740
2741   m = pool_elt_at_index (sm->static_mappings, value.value);
2742
2743   if (by_external)
2744     {
2745       if (is_lb_static_mapping (m))
2746         {
2747           if (PREDICT_FALSE (lb != 0))
2748             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2749           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2750                                                           match.addr,
2751                                                           match.protocol,
2752                                                           match.port,
2753                                                           &backend_index))
2754             {
2755               local = pool_elt_at_index (m->locals, backend_index);
2756               mapping->addr = local->addr;
2757               mapping->port = clib_host_to_net_u16 (local->port);
2758               mapping->fib_index = local->fib_index;
2759               goto end;
2760             }
2761           // pick locals matching this worker
2762           if (PREDICT_FALSE (sm->num_workers > 1))
2763             {
2764               u32 thread_index = vlib_get_thread_index ();
2765               /* *INDENT-OFF* */
2766               pool_foreach_index (i, m->locals,
2767               ({
2768                 local = pool_elt_at_index (m->locals, i);
2769
2770                 ip4_header_t ip = {
2771                   .src_address = local->addr,
2772                 };
2773
2774                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2775                     thread_index)
2776                   {
2777                     vec_add1 (tmp, i);
2778                   }
2779               }));
2780               /* *INDENT-ON* */
2781               ASSERT (vec_len (tmp) != 0);
2782             }
2783           else
2784             {
2785               /* *INDENT-OFF* */
2786               pool_foreach_index (i, m->locals,
2787               ({
2788                 vec_add1 (tmp, i);
2789               }));
2790               /* *INDENT-ON* */
2791             }
2792           hi = vec_len (tmp) - 1;
2793           local = pool_elt_at_index (m->locals, tmp[hi]);
2794           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2795           while (lo < hi)
2796             {
2797               mid = ((hi - lo) >> 1) + lo;
2798               local = pool_elt_at_index (m->locals, tmp[mid]);
2799               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2800             }
2801           local = pool_elt_at_index (m->locals, tmp[lo]);
2802           if (!(local->prefix >= rand))
2803             return 1;
2804           mapping->addr = local->addr;
2805           mapping->port = clib_host_to_net_u16 (local->port);
2806           mapping->fib_index = local->fib_index;
2807           if (m->affinity)
2808             {
2809               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2810                                                 match.protocol, match.port,
2811                                                 tmp[lo], m->affinity,
2812                                                 m->affinity_per_service_list_head_index))
2813                 nat_elog_info ("create affinity record failed");
2814             }
2815           vec_free (tmp);
2816         }
2817       else
2818         {
2819           if (PREDICT_FALSE (lb != 0))
2820             *lb = NO_LB_NAT;
2821           mapping->fib_index = m->fib_index;
2822           mapping->addr = m->local_addr;
2823           /* Address only mapping doesn't change port */
2824           mapping->port = is_addr_only_static_mapping (m) ? match.port
2825             : clib_host_to_net_u16 (m->local_port);
2826         }
2827       mapping->protocol = m->proto;
2828     }
2829   else
2830     {
2831       mapping->addr = m->external_addr;
2832       /* Address only mapping doesn't change port */
2833       mapping->port = is_addr_only_static_mapping (m) ? match.port
2834         : clib_host_to_net_u16 (m->external_port);
2835       mapping->fib_index = sm->outside_fib_index;
2836     }
2837
2838 end:
2839   if (PREDICT_FALSE (is_addr_only != 0))
2840     *is_addr_only = is_addr_only_static_mapping (m);
2841
2842   if (PREDICT_FALSE (twice_nat != 0))
2843     *twice_nat = m->twice_nat;
2844
2845   if (PREDICT_FALSE (is_identity_nat != 0))
2846     *is_identity_nat = is_identity_static_mapping (m);
2847
2848   return 0;
2849 }
2850
2851 static_always_inline u16
2852 snat_random_port (u16 min, u16 max)
2853 {
2854   snat_main_t *sm = &snat_main;
2855   return min + random_u32 (&sm->random_seed) /
2856     (random_u32_max () / (max - min + 1) + 1);
2857 }
2858
2859 int
2860 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2861                                      u32 fib_index,
2862                                      u32 thread_index,
2863                                      snat_session_key_t * k,
2864                                      u16 port_per_thread,
2865                                      u32 snat_thread_index)
2866 {
2867   snat_main_t *sm = &snat_main;
2868
2869   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2870                                   port_per_thread, snat_thread_index);
2871 }
2872
2873 static int
2874 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2875                                  u32 fib_index,
2876                                  u32 thread_index,
2877                                  snat_session_key_t * k,
2878                                  u16 port_per_thread, u32 snat_thread_index)
2879 {
2880   int i;
2881   snat_address_t *a, *ga = 0;
2882   u32 portnum;
2883
2884   for (i = 0; i < vec_len (addresses); i++)
2885     {
2886       a = addresses + i;
2887       switch (k->protocol)
2888         {
2889 #define _(N, j, n, s) \
2890         case SNAT_PROTOCOL_##N: \
2891           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2892             { \
2893               if (a->fib_index == fib_index) \
2894                 { \
2895                   while (1) \
2896                     { \
2897                       portnum = (port_per_thread * \
2898                         snat_thread_index) + \
2899                         snat_random_port(1, port_per_thread) + 1024; \
2900                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2901                         continue; \
2902                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2903                       a->busy_##n##_ports_per_thread[thread_index]++; \
2904                       a->busy_##n##_ports++; \
2905                       k->addr = a->addr; \
2906                       k->port = clib_host_to_net_u16(portnum); \
2907                       return 0; \
2908                     } \
2909                 } \
2910               else if (a->fib_index == ~0) \
2911                 { \
2912                   ga = a; \
2913                 } \
2914             } \
2915           break;
2916           foreach_snat_protocol
2917 #undef _
2918         default:
2919           nat_elog_info ("unknown protocol");
2920           return 1;
2921         }
2922
2923     }
2924
2925   if (ga)
2926     {
2927       a = ga;
2928       switch (k->protocol)
2929         {
2930 #define _(N, j, n, s) \
2931         case SNAT_PROTOCOL_##N: \
2932           while (1) \
2933             { \
2934               portnum = (port_per_thread * \
2935                 snat_thread_index) + \
2936                 snat_random_port(1, port_per_thread) + 1024; \
2937               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2938                 continue; \
2939               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2940               a->busy_##n##_ports_per_thread[thread_index]++; \
2941               a->busy_##n##_ports++; \
2942               k->addr = a->addr; \
2943               k->port = clib_host_to_net_u16(portnum); \
2944               return 0; \
2945             }
2946           break;
2947           foreach_snat_protocol
2948 #undef _
2949         default:
2950           nat_elog_info ("unknown protocol");
2951           return 1;
2952         }
2953     }
2954
2955   /* Totally out of translations to use... */
2956   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2957   return 1;
2958 }
2959
2960 static int
2961 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2962                               u32 fib_index,
2963                               u32 thread_index,
2964                               snat_session_key_t * k,
2965                               u16 port_per_thread, u32 snat_thread_index)
2966 {
2967   snat_main_t *sm = &snat_main;
2968   snat_address_t *a = addresses;
2969   u16 m, ports, portnum, A, j;
2970   m = 16 - (sm->psid_offset + sm->psid_length);
2971   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2972
2973   if (!vec_len (addresses))
2974     goto exhausted;
2975
2976   switch (k->protocol)
2977     {
2978 #define _(N, i, n, s) \
2979     case SNAT_PROTOCOL_##N: \
2980       if (a->busy_##n##_ports < ports) \
2981         { \
2982           while (1) \
2983             { \
2984               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2985               j = snat_random_port(0, pow2_mask(m)); \
2986               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2987               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2988                 continue; \
2989               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2990               a->busy_##n##_ports++; \
2991               k->addr = a->addr; \
2992               k->port = clib_host_to_net_u16 (portnum); \
2993               return 0; \
2994             } \
2995         } \
2996       break;
2997       foreach_snat_protocol
2998 #undef _
2999     default:
3000       nat_elog_info ("unknown protocol");
3001       return 1;
3002     }
3003
3004 exhausted:
3005   /* Totally out of translations to use... */
3006   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3007   return 1;
3008 }
3009
3010 static int
3011 nat_alloc_addr_and_port_range (snat_address_t * addresses,
3012                                u32 fib_index,
3013                                u32 thread_index,
3014                                snat_session_key_t * k,
3015                                u16 port_per_thread, u32 snat_thread_index)
3016 {
3017   snat_main_t *sm = &snat_main;
3018   snat_address_t *a = addresses;
3019   u16 portnum, ports;
3020
3021   ports = sm->end_port - sm->start_port + 1;
3022
3023   if (!vec_len (addresses))
3024     goto exhausted;
3025
3026   switch (k->protocol)
3027     {
3028 #define _(N, i, n, s) \
3029     case SNAT_PROTOCOL_##N: \
3030       if (a->busy_##n##_ports < ports) \
3031         { \
3032           while (1) \
3033             { \
3034               portnum = snat_random_port(sm->start_port, sm->end_port); \
3035               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
3036                 continue; \
3037               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
3038               a->busy_##n##_ports++; \
3039               k->addr = a->addr; \
3040               k->port = clib_host_to_net_u16 (portnum); \
3041               return 0; \
3042             } \
3043         } \
3044       break;
3045       foreach_snat_protocol
3046 #undef _
3047     default:
3048       nat_elog_info ("unknown protocol");
3049       return 1;
3050     }
3051
3052 exhausted:
3053   /* Totally out of translations to use... */
3054   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3055   return 1;
3056 }
3057
3058 void
3059 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3060 {
3061   dpo_id_t dpo_v4 = DPO_INVALID;
3062   fib_prefix_t pfx = {
3063     .fp_proto = FIB_PROTOCOL_IP4,
3064     .fp_len = 32,
3065     .fp_addr.ip4.as_u32 = addr.as_u32,
3066   };
3067
3068   if (is_add)
3069     {
3070       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3071       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3072                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3073       dpo_reset (&dpo_v4);
3074     }
3075   else
3076     {
3077       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3078     }
3079 }
3080
3081 u8 *
3082 format_session_kvp (u8 * s, va_list * args)
3083 {
3084   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3085   snat_session_key_t k;
3086
3087   k.as_u64 = v->key;
3088
3089   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
3090
3091   return s;
3092 }
3093
3094 u8 *
3095 format_static_mapping_kvp (u8 * s, va_list * args)
3096 {
3097   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3098   snat_session_key_t k;
3099
3100   k.as_u64 = v->key;
3101
3102   s = format (s, "%U static-mapping-index %llu",
3103               format_static_mapping_key, &k, v->value);
3104
3105   return s;
3106 }
3107
3108 u8 *
3109 format_user_kvp (u8 * s, va_list * args)
3110 {
3111   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3112   snat_user_key_t k;
3113
3114   k.as_u64 = v->key;
3115
3116   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3117               k.fib_index, v->value);
3118
3119   return s;
3120 }
3121
3122 u8 *
3123 format_ed_session_kvp (u8 * s, va_list * args)
3124 {
3125   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3126   nat_ed_ses_key_t k;
3127
3128   k.as_u64[0] = v->key[0];
3129   k.as_u64[1] = v->key[1];
3130
3131   s =
3132     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3133             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
3134             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
3135             format_ip_protocol, k.proto, k.fib_index, v->value);
3136
3137   return s;
3138 }
3139
3140 static u32
3141 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3142                            u8 is_output)
3143 {
3144   snat_main_t *sm = &snat_main;
3145   u32 next_worker_index = 0;
3146   u32 hash;
3147
3148   next_worker_index = sm->first_worker_index;
3149   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3150     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3151
3152   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3153     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3154   else
3155     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3156
3157   return next_worker_index;
3158 }
3159
3160 static u32
3161 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3162                            u32 rx_fib_index0, u8 is_output)
3163 {
3164   snat_main_t *sm = &snat_main;
3165   udp_header_t *udp;
3166   u16 port;
3167   snat_session_key_t m_key;
3168   clib_bihash_kv_8_8_t kv, value;
3169   snat_static_mapping_t *m;
3170   u32 proto;
3171   u32 next_worker_index = 0;
3172
3173   /* first try static mappings without port */
3174   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3175     {
3176       m_key.addr = ip0->dst_address;
3177       m_key.port = 0;
3178       m_key.protocol = 0;
3179       m_key.fib_index = rx_fib_index0;
3180       kv.key = m_key.as_u64;
3181       if (!clib_bihash_search_8_8
3182           (&sm->static_mapping_by_external, &kv, &value))
3183         {
3184           m = pool_elt_at_index (sm->static_mappings, value.value);
3185           return m->workers[0];
3186         }
3187     }
3188
3189   proto = ip_proto_to_snat_proto (ip0->protocol);
3190   udp = ip4_next_header (ip0);
3191   port = udp->dst_port;
3192
3193   /* unknown protocol */
3194   if (PREDICT_FALSE (proto == ~0))
3195     {
3196       /* use current thread */
3197       return vlib_get_thread_index ();
3198     }
3199
3200   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3201     {
3202       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3203       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3204       if (!icmp_type_is_error_message
3205           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3206         port = vnet_buffer (b)->ip.reass.l4_src_port;
3207       else
3208         {
3209           /* if error message, then it's not fragmented and we can access it */
3210           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3211           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3212           void *l4_header = ip4_next_header (inner_ip);
3213           switch (proto)
3214             {
3215             case SNAT_PROTOCOL_ICMP:
3216               icmp = (icmp46_header_t *) l4_header;
3217               echo = (icmp_echo_header_t *) (icmp + 1);
3218               port = echo->identifier;
3219               break;
3220             case SNAT_PROTOCOL_UDP:
3221             case SNAT_PROTOCOL_TCP:
3222               port = ((tcp_udp_header_t *) l4_header)->src_port;
3223               break;
3224             default:
3225               return vlib_get_thread_index ();
3226             }
3227         }
3228     }
3229
3230   /* try static mappings with port */
3231   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3232     {
3233       m_key.addr = ip0->dst_address;
3234       m_key.port = clib_net_to_host_u16 (port);
3235       m_key.protocol = proto;
3236       m_key.fib_index = rx_fib_index0;
3237       kv.key = m_key.as_u64;
3238       if (!clib_bihash_search_8_8
3239           (&sm->static_mapping_by_external, &kv, &value))
3240         {
3241           m = pool_elt_at_index (sm->static_mappings, value.value);
3242           return m->workers[0];
3243         }
3244     }
3245
3246   /* worker by outside port */
3247   next_worker_index = sm->first_worker_index;
3248   next_worker_index +=
3249     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3250   return next_worker_index;
3251 }
3252
3253 static u32
3254 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3255                                u8 is_output)
3256 {
3257   snat_main_t *sm = &snat_main;
3258   u32 next_worker_index = sm->first_worker_index;
3259   u32 hash;
3260
3261   clib_bihash_kv_16_8_t kv16, value16;
3262   snat_main_per_thread_data_t *tsm;
3263   udp_header_t *udp;
3264
3265   if (PREDICT_FALSE (is_output))
3266     {
3267       u32 fib_index = sm->outside_fib_index;
3268       nat_outside_fib_t *outside_fib;
3269       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3270       fib_prefix_t pfx = {
3271         .fp_proto = FIB_PROTOCOL_IP4,
3272         .fp_len = 32,
3273         .fp_addr = {
3274                     .ip4.as_u32 = ip->dst_address.as_u32,
3275                     }
3276         ,
3277       };
3278
3279       udp = ip4_next_header (ip);
3280
3281       switch (vec_len (sm->outside_fibs))
3282         {
3283         case 0:
3284           fib_index = sm->outside_fib_index;
3285           break;
3286         case 1:
3287           fib_index = sm->outside_fibs[0].fib_index;
3288           break;
3289         default:
3290             /* *INDENT-OFF* */
3291             vec_foreach (outside_fib, sm->outside_fibs)
3292               {
3293                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3294                 if (FIB_NODE_INDEX_INVALID != fei)
3295                   {
3296                     if (fib_entry_get_resolving_interface (fei) != ~0)
3297                       {
3298                         fib_index = outside_fib->fib_index;
3299                         break;
3300                       }
3301                   }
3302               }
3303             /* *INDENT-ON* */
3304           break;
3305         }
3306
3307       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3308                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3309
3310       /* *INDENT-OFF* */
3311       vec_foreach (tsm, sm->per_thread_data)
3312         {
3313           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3314                                                       &kv16, &value16)))
3315             {
3316               next_worker_index += tsm->thread_index;
3317
3318               nat_elog_debug_handoff (
3319                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3320                 next_worker_index, fib_index,
3321                 clib_net_to_host_u32 (ip->src_address.as_u32),
3322                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3323
3324               return next_worker_index;
3325             }
3326         }
3327       /* *INDENT-ON* */
3328     }
3329
3330   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3331     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3332
3333   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3334     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3335   else
3336     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3337
3338   if (PREDICT_TRUE (!is_output))
3339     {
3340       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3341                               next_worker_index, rx_fib_index,
3342                               clib_net_to_host_u32 (ip->src_address.as_u32),
3343                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3344     }
3345   else
3346     {
3347       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3348                               next_worker_index, rx_fib_index,
3349                               clib_net_to_host_u32 (ip->src_address.as_u32),
3350                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3351     }
3352
3353   return next_worker_index;
3354 }
3355
3356 static u32
3357 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3358                                u32 rx_fib_index, u8 is_output)
3359 {
3360   snat_main_t *sm = &snat_main;
3361   clib_bihash_kv_8_8_t kv, value;
3362   clib_bihash_kv_16_8_t kv16, value16;
3363   snat_main_per_thread_data_t *tsm;
3364
3365   u32 proto, next_worker_index = 0;
3366   udp_header_t *udp;
3367   u16 port;
3368   snat_static_mapping_t *m;
3369   u32 hash;
3370
3371   proto = ip_proto_to_snat_proto (ip->protocol);
3372
3373   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3374     {
3375       udp = ip4_next_header (ip);
3376
3377       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3378                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3379
3380       /* *INDENT-OFF* */
3381       vec_foreach (tsm, sm->per_thread_data)
3382         {
3383           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3384                                                       &kv16, &value16)))
3385             {
3386               next_worker_index = sm->first_worker_index + tsm->thread_index;
3387               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3388                           next_worker_index, rx_fib_index,
3389                           clib_net_to_host_u32 (ip->src_address.as_u32),
3390                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3391               return next_worker_index;
3392             }
3393           }
3394         /* *INDENT-ON* */
3395     }
3396   else if (proto == SNAT_PROTOCOL_ICMP)
3397     {
3398       nat_ed_ses_key_t key;
3399
3400       if (!get_icmp_o2i_ed_key (b, ip, &key))
3401         {
3402
3403           key.fib_index = rx_fib_index;
3404           kv16.key[0] = key.as_u64[0];
3405           kv16.key[1] = key.as_u64[1];
3406
3407           /* *INDENT-OFF* */
3408           vec_foreach (tsm, sm->per_thread_data)
3409             {
3410               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3411                                                           &kv16, &value16)))
3412                 {
3413                   next_worker_index = sm->first_worker_index +
3414                                       tsm->thread_index;
3415                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3416                               next_worker_index, rx_fib_index,
3417                               clib_net_to_host_u32 (ip->src_address.as_u32),
3418                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3419                   return next_worker_index;
3420                 }
3421             }
3422           /* *INDENT-ON* */
3423         }
3424     }
3425
3426   /* first try static mappings without port */
3427   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3428     {
3429       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3430       if (!clib_bihash_search_8_8
3431           (&sm->static_mapping_by_external, &kv, &value))
3432         {
3433           m = pool_elt_at_index (sm->static_mappings, value.value);
3434           next_worker_index = m->workers[0];
3435           goto done;
3436         }
3437     }
3438
3439   /* unknown protocol */
3440   if (PREDICT_FALSE (proto == ~0))
3441     {
3442       /* use current thread */
3443       next_worker_index = vlib_get_thread_index ();
3444       goto done;
3445     }
3446
3447   udp = ip4_next_header (ip);
3448   port = udp->dst_port;
3449
3450   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3451     {
3452       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3453       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3454       if (!icmp_type_is_error_message
3455           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3456         port = vnet_buffer (b)->ip.reass.l4_src_port;
3457       else
3458         {
3459           /* if error message, then it's not fragmented and we can access it */
3460           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3461           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3462           void *l4_header = ip4_next_header (inner_ip);
3463           switch (proto)
3464             {
3465             case SNAT_PROTOCOL_ICMP:
3466               icmp = (icmp46_header_t *) l4_header;
3467               echo = (icmp_echo_header_t *) (icmp + 1);
3468               port = echo->identifier;
3469               break;
3470             case SNAT_PROTOCOL_UDP:
3471             case SNAT_PROTOCOL_TCP:
3472               port = ((tcp_udp_header_t *) l4_header)->src_port;
3473               break;
3474             default:
3475               next_worker_index = vlib_get_thread_index ();
3476               goto done;
3477             }
3478         }
3479     }
3480
3481   /* try static mappings with port */
3482   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3483     {
3484       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3485                   clib_net_to_host_u16 (port));
3486       if (!clib_bihash_search_8_8
3487           (&sm->static_mapping_by_external, &kv, &value))
3488         {
3489           m = pool_elt_at_index (sm->static_mappings, value.value);
3490           if (!is_lb_static_mapping (m))
3491             {
3492               next_worker_index = m->workers[0];
3493               goto done;
3494             }
3495
3496           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3497             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3498
3499           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3500             next_worker_index =
3501               m->workers[hash & (_vec_len (m->workers) - 1)];
3502           else
3503             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3504           goto done;
3505         }
3506     }
3507
3508   /* worker by outside port */
3509   next_worker_index = sm->first_worker_index;
3510   next_worker_index +=
3511     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3512
3513 done:
3514   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3515                           clib_net_to_host_u32 (ip->src_address.as_u32),
3516                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3517   return next_worker_index;
3518 }
3519
3520 void
3521 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3522                 ip4_address_t * out_addr, u16 out_port,
3523                 ip4_address_t * eh_addr, u16 eh_port,
3524                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3525                 u32 fib_index, u16 flags, u32 thread_index)
3526 {
3527   snat_main_t *sm = &snat_main;
3528   snat_session_key_t key;
3529   snat_user_t *u;
3530   snat_session_t *s;
3531   clib_bihash_kv_8_8_t kv;
3532   f64 now = vlib_time_now (sm->vlib_main);
3533   nat_outside_fib_t *outside_fib;
3534   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3535   snat_main_per_thread_data_t *tsm;
3536   fib_prefix_t pfx = {
3537     .fp_proto = FIB_PROTOCOL_IP4,
3538     .fp_len = 32,
3539     .fp_addr = {
3540                 .ip4.as_u32 = eh_addr->as_u32,
3541                 },
3542   };
3543
3544   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3545
3546   key.addr.as_u32 = out_addr->as_u32;
3547   key.port = out_port;
3548   key.protocol = proto;
3549
3550   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3551     {
3552       if (nat_set_outside_address_and_port
3553           (sm->addresses, thread_index, &key))
3554         return;
3555     }
3556
3557   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3558   if (!u)
3559     return;
3560
3561   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3562   if (!s)
3563     return;
3564
3565   s->last_heard = now;
3566   s->flags = flags;
3567   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3568   s->ext_host_port = eh_port;
3569   user_session_increment (sm, u, snat_is_session_static (s));
3570   switch (vec_len (sm->outside_fibs))
3571     {
3572     case 0:
3573       key.fib_index = sm->outside_fib_index;
3574       break;
3575     case 1:
3576       key.fib_index = sm->outside_fibs[0].fib_index;
3577       break;
3578     default:
3579       /* *INDENT-OFF* */
3580       vec_foreach (outside_fib, sm->outside_fibs)
3581         {
3582           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3583           if (FIB_NODE_INDEX_INVALID != fei)
3584             {
3585               if (fib_entry_get_resolving_interface (fei) != ~0)
3586                 {
3587                   key.fib_index = outside_fib->fib_index;
3588                   break;
3589                 }
3590             }
3591         }
3592       /* *INDENT-ON* */
3593       break;
3594     }
3595   s->out2in = key;
3596   kv.key = key.as_u64;
3597   kv.value = s - tsm->sessions;
3598   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3599     nat_elog_warn ("out2in key add failed");
3600
3601   key.addr.as_u32 = in_addr->as_u32;
3602   key.port = in_port;
3603   key.fib_index = fib_index;
3604   s->in2out = key;
3605   kv.key = key.as_u64;
3606   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3607     nat_elog_warn ("in2out key add failed");
3608 }
3609
3610 void
3611 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3612                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3613                 u32 ti)
3614 {
3615   snat_main_t *sm = &snat_main;
3616   snat_session_key_t key;
3617   clib_bihash_kv_8_8_t kv, value;
3618   u32 thread_index;
3619   snat_session_t *s;
3620   snat_main_per_thread_data_t *tsm;
3621
3622   if (sm->num_workers > 1)
3623     thread_index =
3624       sm->first_worker_index +
3625       (sm->workers[(clib_net_to_host_u16 (out_port) -
3626                     1024) / sm->port_per_thread]);
3627   else
3628     thread_index = sm->num_workers;
3629   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3630
3631   key.addr.as_u32 = out_addr->as_u32;
3632   key.port = out_port;
3633   key.protocol = proto;
3634   key.fib_index = fib_index;
3635   kv.key = key.as_u64;
3636   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3637     return;
3638
3639   s = pool_elt_at_index (tsm->sessions, value.value);
3640   nat_free_session_data (sm, s, thread_index, 1);
3641   nat44_delete_session (sm, s, thread_index);
3642 }
3643
3644 void
3645 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3646                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3647                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3648 {
3649   snat_main_t *sm = &snat_main;
3650   snat_session_key_t key;
3651   clib_bihash_kv_8_8_t kv, value;
3652   snat_session_t *s;
3653   snat_main_per_thread_data_t *tsm;
3654
3655   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3656
3657   key.addr.as_u32 = out_addr->as_u32;
3658   key.port = out_port;
3659   key.protocol = proto;
3660   key.fib_index = fib_index;
3661   kv.key = key.as_u64;
3662   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3663     return;
3664
3665   s = pool_elt_at_index (tsm->sessions, value.value);
3666   s->total_pkts = total_pkts;
3667   s->total_bytes = total_bytes;
3668 }
3669
3670 void
3671 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3672                    ip4_address_t * out_addr, u16 out_port,
3673                    ip4_address_t * eh_addr, u16 eh_port,
3674                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3675                    u32 fib_index, u16 flags, u32 thread_index)
3676 {
3677   snat_main_t *sm = &snat_main;
3678   snat_session_key_t key;
3679   snat_user_t *u;
3680   snat_session_t *s;
3681   clib_bihash_kv_16_8_t kv;
3682   f64 now = vlib_time_now (sm->vlib_main);
3683   nat_outside_fib_t *outside_fib;
3684   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3685   snat_main_per_thread_data_t *tsm;
3686   fib_prefix_t pfx = {
3687     .fp_proto = FIB_PROTOCOL_IP4,
3688     .fp_len = 32,
3689     .fp_addr = {
3690                 .ip4.as_u32 = eh_addr->as_u32,
3691                 },
3692   };
3693
3694   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3695
3696   key.addr.as_u32 = out_addr->as_u32;
3697   key.port = out_port;
3698   key.protocol = proto;
3699
3700   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3701     {
3702       if (nat_set_outside_address_and_port
3703           (sm->addresses, thread_index, &key))
3704         return;
3705     }
3706
3707   key.addr.as_u32 = ehn_addr->as_u32;
3708   key.port = ehn_port;
3709   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3710     {
3711       if (nat_set_outside_address_and_port
3712           (sm->twice_nat_addresses, thread_index, &key))
3713         return;
3714     }
3715
3716   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3717   if (!u)
3718     return;
3719
3720   s = nat_ed_session_alloc (sm, u, thread_index, now);
3721   if (!s)
3722     return;
3723
3724   s->last_heard = now;
3725   s->flags = flags;
3726   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3727   s->ext_host_nat_port = s->ext_host_port = eh_port;
3728   if (is_twice_nat_session (s))
3729     {
3730       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3731       s->ext_host_nat_port = ehn_port;
3732     }
3733   user_session_increment (sm, u, snat_is_session_static (s));
3734   switch (vec_len (sm->outside_fibs))
3735     {
3736     case 0:
3737       key.fib_index = sm->outside_fib_index;
3738       break;
3739     case 1:
3740       key.fib_index = sm->outside_fibs[0].fib_index;
3741       break;
3742     default:
3743       /* *INDENT-OFF* */
3744       vec_foreach (outside_fib, sm->outside_fibs)
3745         {
3746           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3747           if (FIB_NODE_INDEX_INVALID != fei)
3748             {
3749               if (fib_entry_get_resolving_interface (fei) != ~0)
3750                 {
3751                   key.fib_index = outside_fib->fib_index;
3752                   break;
3753                 }
3754             }
3755         }
3756       /* *INDENT-ON* */
3757       break;
3758     }
3759   key.addr.as_u32 = out_addr->as_u32;
3760   key.port = out_port;
3761   s->out2in = key;
3762   kv.value = s - tsm->sessions;
3763
3764   key.addr.as_u32 = in_addr->as_u32;
3765   key.port = in_port;
3766   key.fib_index = fib_index;
3767   s->in2out = key;
3768
3769   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3770               snat_proto_to_ip_proto (proto), fib_index, in_port,
3771               s->ext_host_nat_port);
3772   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3773     nat_elog_warn ("in2out key add failed");
3774
3775   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3776               s->out2in.fib_index, out_port, eh_port);
3777   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3778     nat_elog_warn ("out2in key add failed");
3779 }
3780
3781 void
3782 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3783                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3784                    u32 fib_index, u32 ti)
3785 {
3786   snat_main_t *sm = &snat_main;
3787   nat_ed_ses_key_t key;
3788   clib_bihash_kv_16_8_t kv, value;
3789   u32 thread_index;
3790   snat_session_t *s;
3791   snat_main_per_thread_data_t *tsm;
3792
3793   if (sm->num_workers > 1)
3794     thread_index =
3795       sm->first_worker_index +
3796       (sm->workers[(clib_net_to_host_u16 (out_port) -
3797                     1024) / sm->port_per_thread]);
3798   else
3799     thread_index = sm->num_workers;
3800   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3801
3802   key.l_addr.as_u32 = out_addr->as_u32;
3803   key.l_port = out_port;
3804   key.r_addr.as_u32 = eh_addr->as_u32;
3805   key.r_port = eh_port;
3806   key.proto = proto;
3807   key.fib_index = fib_index;
3808   kv.key[0] = key.as_u64[0];
3809   kv.key[1] = key.as_u64[1];
3810   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3811     return;
3812
3813   s = pool_elt_at_index (tsm->sessions, value.value);
3814   nat_free_session_data (sm, s, thread_index, 1);
3815   nat44_delete_session (sm, s, thread_index);
3816 }
3817
3818 void
3819 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3820                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3821                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3822                    u32 thread_index)
3823 {
3824   snat_main_t *sm = &snat_main;
3825   nat_ed_ses_key_t key;
3826   clib_bihash_kv_16_8_t kv, value;
3827   snat_session_t *s;
3828   snat_main_per_thread_data_t *tsm;
3829
3830   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3831
3832   key.l_addr.as_u32 = out_addr->as_u32;
3833   key.l_port = out_port;
3834   key.r_addr.as_u32 = eh_addr->as_u32;
3835   key.r_port = eh_port;
3836   key.proto = proto;
3837   key.fib_index = fib_index;
3838   kv.key[0] = key.as_u64[0];
3839   kv.key[1] = key.as_u64[1];
3840   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3841     return;
3842
3843   s = pool_elt_at_index (tsm->sessions, value.value);
3844   s->total_pkts = total_pkts;
3845   s->total_bytes = total_bytes;
3846 }
3847
3848 static clib_error_t *
3849 snat_config (vlib_main_t * vm, unformat_input_t * input)
3850 {
3851   snat_main_t *sm = &snat_main;
3852   nat66_main_t *nm = &nat66_main;
3853   //dslite_main_t *dm = &dslite_main;
3854   snat_main_per_thread_data_t *tsm;
3855
3856   u32 static_mapping_buckets = 1024;
3857   u32 static_mapping_memory_size = 64 << 20;
3858
3859   u32 nat64_bib_buckets = 1024;
3860   u32 nat64_bib_memory_size = 128 << 20;
3861
3862   u32 nat64_st_buckets = 2048;
3863   u32 nat64_st_memory_size = 256 << 20;
3864
3865   u32 user_buckets = 128;
3866   u32 user_memory_size = 64 << 20;
3867   u32 translation_buckets = 1024;
3868   u32 translation_memory_size = 128 << 20;
3869
3870   u32 max_translations_per_user = ~0;
3871
3872   u32 outside_vrf_id = 0;
3873   u32 outside_ip6_vrf_id = 0;
3874   u32 inside_vrf_id = 0;
3875   u8 static_mapping_only = 0;
3876   u8 static_mapping_connection_tracking = 0;
3877
3878   // configurable timeouts
3879   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3880   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3881   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3882   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3883
3884   sm->deterministic = 0;
3885   sm->out2in_dpo = 0;
3886   sm->endpoint_dependent = 0;
3887
3888   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3889     {
3890       if (unformat
3891           (input, "translation hash buckets %d", &translation_buckets))
3892         ;
3893       else if (unformat (input, "udp timeout %d", &udp_timeout))
3894         ;
3895       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3896         ;
3897       else if (unformat (input, "tcp transitory timeout %d",
3898                          &tcp_transitory_timeout));
3899       else if (unformat (input, "tcp established timeout %d",
3900                          &tcp_established_timeout));
3901       else if (unformat (input, "translation hash memory %d",
3902                          &translation_memory_size));
3903       else if (unformat (input, "user hash buckets %d", &user_buckets))
3904         ;
3905       else if (unformat (input, "user hash memory %d", &user_memory_size))
3906         ;
3907       else if (unformat (input, "max translations per user %d",
3908                          &max_translations_per_user))
3909         ;
3910       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3911         ;
3912       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3913         ;
3914       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3915         ;
3916       else if (unformat (input, "static mapping only"))
3917         {
3918           static_mapping_only = 1;
3919           if (unformat (input, "connection tracking"))
3920             static_mapping_connection_tracking = 1;
3921         }
3922       else if (unformat (input, "deterministic"))
3923         sm->deterministic = 1;
3924       else if (unformat (input, "nat64 bib hash buckets %d",
3925                          &nat64_bib_buckets))
3926         ;
3927       else if (unformat (input, "nat64 bib hash memory %d",
3928                          &nat64_bib_memory_size))
3929         ;
3930       else
3931         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3932         ;
3933       else if (unformat (input, "nat64 st hash memory %d",
3934                          &nat64_st_memory_size))
3935         ;
3936       else if (unformat (input, "out2in dpo"))
3937         sm->out2in_dpo = 1;
3938       //else if (unformat (input, "dslite ce"))
3939       //dslite_set_ce (dm, 1);
3940       else if (unformat (input, "endpoint-dependent"))
3941         sm->endpoint_dependent = 1;
3942       else
3943         return clib_error_return (0, "unknown input '%U'",
3944                                   format_unformat_error, input);
3945     }
3946
3947   if (sm->deterministic && sm->endpoint_dependent)
3948     return clib_error_return (0,
3949                               "deterministic and endpoint-dependent modes are mutually exclusive");
3950
3951   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3952     return clib_error_return (0,
3953                               "static mapping only mode available only for simple nat");
3954
3955   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3956     return clib_error_return (0,
3957                               "out2in dpo mode available only for simple nat");
3958
3959   /* optionally configurable timeouts for testing purposes */
3960   sm->udp_timeout = udp_timeout;
3961   sm->tcp_transitory_timeout = tcp_transitory_timeout;
3962   sm->tcp_established_timeout = tcp_established_timeout;
3963   sm->icmp_timeout = icmp_timeout;
3964
3965   sm->min_timeout = nat44_minimal_timeout (sm);
3966
3967   sm->user_buckets = user_buckets;
3968   sm->user_memory_size = user_memory_size;
3969
3970   sm->translation_buckets = translation_buckets;
3971   sm->translation_memory_size = translation_memory_size;
3972
3973   /* do not exceed load factor 10 */
3974   sm->max_translations = 10 * translation_buckets;
3975   sm->max_translations_per_user = max_translations_per_user == ~0 ?
3976     sm->max_translations : max_translations_per_user;
3977
3978   sm->outside_vrf_id = outside_vrf_id;
3979   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3980                                                              outside_vrf_id,
3981                                                              nat_fib_src_hi);
3982   nm->outside_vrf_id = outside_ip6_vrf_id;
3983   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3984                                                              outside_ip6_vrf_id,
3985                                                              nat_fib_src_hi);
3986   sm->inside_vrf_id = inside_vrf_id;
3987   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3988                                                             inside_vrf_id,
3989                                                             nat_fib_src_hi);
3990   sm->static_mapping_only = static_mapping_only;
3991   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3992
3993   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3994                   nat64_st_memory_size);
3995
3996   if (sm->deterministic)
3997     {
3998       sm->in2out_node_index = snat_det_in2out_node.index;
3999       sm->in2out_output_node_index = ~0;
4000       sm->out2in_node_index = snat_det_out2in_node.index;
4001       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4002       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4003     }
4004   else
4005     {
4006       if (sm->endpoint_dependent)
4007         {
4008           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4009           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4010
4011           sm->handoff_out2in_index = nat_pre_out2in_node.index;
4012           sm->handoff_in2out_index = nat_pre_in2out_node.index;
4013           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
4014
4015           sm->in2out_node_index = nat44_ed_in2out_node.index;
4016           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4017           sm->out2in_node_index = nat44_ed_out2in_node.index;
4018
4019           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4020           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4021           nat_affinity_init (vm);
4022           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4023                        nat_ha_sref_ed_cb);
4024         }
4025       else
4026         {
4027           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4028           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4029
4030           sm->handoff_out2in_index = snat_in2out_node.index;
4031           sm->handoff_in2out_index = snat_out2in_node.index;
4032           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
4033
4034           sm->in2out_node_index = snat_in2out_node.index;
4035           sm->in2out_output_node_index = snat_in2out_output_node.index;
4036           sm->out2in_node_index = snat_out2in_node.index;
4037           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4038           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4039           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4040         }
4041       if (!static_mapping_only ||
4042           (static_mapping_only && static_mapping_connection_tracking))
4043         {
4044           /* *INDENT-OFF* */
4045           vec_foreach (tsm, sm->per_thread_data)
4046             {
4047               tsm->min_session_timeout = 0;
4048
4049               tsm->cleared = 0;
4050               tsm->cleanup_runs = 0;
4051               tsm->cleanup_timeout = 0;
4052
4053               if (sm->endpoint_dependent)
4054                 {
4055                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
4056                                          translation_buckets,
4057                                          translation_memory_size);
4058                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
4059                                                       format_ed_session_kvp);
4060
4061                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
4062                                          translation_buckets,
4063                                          translation_memory_size);
4064                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
4065                                                       format_ed_session_kvp);
4066                 }
4067               else
4068                 {
4069                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
4070                                         translation_buckets,
4071                                         translation_memory_size);
4072                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
4073                                                      format_session_kvp);
4074
4075                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
4076                                         translation_buckets,
4077                                         translation_memory_size);
4078                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
4079                                                      format_session_kvp);
4080                 }
4081
4082               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
4083                                     user_memory_size);
4084               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
4085                                                  format_user_kvp);
4086             }
4087           /* *INDENT-ON* */
4088
4089         }
4090       else
4091         {
4092           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4093           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4094         }
4095       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4096                             "static_mapping_by_local", static_mapping_buckets,
4097                             static_mapping_memory_size);
4098       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4099                                          format_static_mapping_kvp);
4100
4101       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4102                             "static_mapping_by_external",
4103                             static_mapping_buckets,
4104                             static_mapping_memory_size);
4105       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4106                                          format_static_mapping_kvp);
4107     }
4108
4109   return 0;
4110 }
4111
4112 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4113
4114 static void
4115 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4116                                  uword opaque,
4117                                  u32 sw_if_index,
4118                                  ip4_address_t * address,
4119                                  u32 address_length,
4120                                  u32 if_address_index, u32 is_delete)
4121 {
4122   snat_main_t *sm = &snat_main;
4123   snat_static_map_resolve_t *rp;
4124   snat_static_mapping_t *m;
4125   snat_session_key_t m_key;
4126   clib_bihash_kv_8_8_t kv, value;
4127   int i, rv;
4128   ip4_address_t l_addr;
4129
4130   for (i = 0; i < vec_len (sm->to_resolve); i++)
4131     {
4132       rp = sm->to_resolve + i;
4133       if (rp->addr_only == 0)
4134         continue;
4135       if (rp->sw_if_index == sw_if_index)
4136         goto match;
4137     }
4138
4139   return;
4140
4141 match:
4142   m_key.addr.as_u32 = address->as_u32;
4143   m_key.port = rp->addr_only ? 0 : rp->e_port;
4144   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4145   m_key.fib_index = sm->outside_fib_index;
4146   kv.key = m_key.as_u64;
4147   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4148     m = 0;
4149   else
4150     m = pool_elt_at_index (sm->static_mappings, value.value);
4151
4152   if (!is_delete)
4153     {
4154       /* Don't trip over lease renewal, static config */
4155       if (m)
4156         return;
4157     }
4158   else
4159     {
4160       if (!m)
4161         return;
4162     }
4163
4164   /* Indetity mapping? */
4165   if (rp->l_addr.as_u32 == 0)
4166     l_addr.as_u32 = address[0].as_u32;
4167   else
4168     l_addr.as_u32 = rp->l_addr.as_u32;
4169   /* Add the static mapping */
4170   rv = snat_add_static_mapping (l_addr,
4171                                 address[0],
4172                                 rp->l_port,
4173                                 rp->e_port,
4174                                 rp->vrf_id,
4175                                 rp->addr_only, ~0 /* sw_if_index */ ,
4176                                 rp->proto, !is_delete, rp->twice_nat,
4177                                 rp->out2in_only, rp->tag, rp->identity_nat);
4178   if (rv)
4179     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4180 }
4181
4182 static void
4183 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4184                                        uword opaque,
4185                                        u32 sw_if_index,
4186                                        ip4_address_t * address,
4187                                        u32 address_length,
4188                                        u32 if_address_index, u32 is_delete)
4189 {
4190   snat_main_t *sm = &snat_main;
4191   snat_static_map_resolve_t *rp;
4192   ip4_address_t l_addr;
4193   int i, j;
4194   int rv;
4195   u8 twice_nat = 0;
4196   snat_address_t *addresses = sm->addresses;
4197
4198   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4199     {
4200       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4201         goto match;
4202     }
4203
4204   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4205     {
4206       twice_nat = 1;
4207       addresses = sm->twice_nat_addresses;
4208       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4209         goto match;
4210     }
4211
4212   return;
4213
4214 match:
4215   if (!is_delete)
4216     {
4217       /* Don't trip over lease renewal, static config */
4218       for (j = 0; j < vec_len (addresses); j++)
4219         if (addresses[j].addr.as_u32 == address->as_u32)
4220           return;
4221
4222       (void) snat_add_address (sm, address, ~0, twice_nat);
4223       /* Scan static map resolution vector */
4224       for (j = 0; j < vec_len (sm->to_resolve); j++)
4225         {
4226           rp = sm->to_resolve + j;
4227           if (rp->addr_only)
4228             continue;
4229           /* On this interface? */
4230           if (rp->sw_if_index == sw_if_index)
4231             {
4232               /* Indetity mapping? */
4233               if (rp->l_addr.as_u32 == 0)
4234                 l_addr.as_u32 = address[0].as_u32;
4235               else
4236                 l_addr.as_u32 = rp->l_addr.as_u32;
4237               /* Add the static mapping */
4238               rv = snat_add_static_mapping (l_addr,
4239                                             address[0],
4240                                             rp->l_port,
4241                                             rp->e_port,
4242                                             rp->vrf_id,
4243                                             rp->addr_only,
4244                                             ~0 /* sw_if_index */ ,
4245                                             rp->proto,
4246                                             rp->is_add, rp->twice_nat,
4247                                             rp->out2in_only, rp->tag,
4248                                             rp->identity_nat);
4249               if (rv)
4250                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4251                                     "i4", rv);
4252             }
4253         }
4254       return;
4255     }
4256   else
4257     {
4258       (void) snat_del_address (sm, address[0], 1, twice_nat);
4259       return;
4260     }
4261 }
4262
4263
4264 int
4265 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4266                             u8 twice_nat)
4267 {
4268   ip4_main_t *ip4_main = sm->ip4_main;
4269   ip4_address_t *first_int_addr;
4270   snat_static_map_resolve_t *rp;
4271   u32 *indices_to_delete = 0;
4272   int i, j;
4273   u32 *auto_add_sw_if_indices =
4274     twice_nat ? sm->
4275     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4276
4277   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4278     );
4279
4280   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4281     {
4282       if (auto_add_sw_if_indices[i] == sw_if_index)
4283         {
4284           if (is_del)
4285             {
4286               /* if have address remove it */
4287               if (first_int_addr)
4288                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4289               else
4290                 {
4291                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4292                     {
4293                       rp = sm->to_resolve + j;
4294                       if (rp->sw_if_index == sw_if_index)
4295                         vec_add1 (indices_to_delete, j);
4296                     }
4297                   if (vec_len (indices_to_delete))
4298                     {
4299                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4300                         vec_del1 (sm->to_resolve, j);
4301                       vec_free (indices_to_delete);
4302                     }
4303                 }
4304               if (twice_nat)
4305                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4306               else
4307                 vec_del1 (sm->auto_add_sw_if_indices, i);
4308             }
4309           else
4310             return VNET_API_ERROR_VALUE_EXIST;
4311
4312           return 0;
4313         }
4314     }
4315
4316   if (is_del)
4317     return VNET_API_ERROR_NO_SUCH_ENTRY;
4318
4319   /* add to the auto-address list */
4320   if (twice_nat)
4321     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4322   else
4323     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4324
4325   /* If the address is already bound - or static - add it now */
4326   if (first_int_addr)
4327     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4328
4329   return 0;
4330 }
4331
4332 int
4333 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4334                    snat_protocol_t proto, u32 vrf_id, int is_in)
4335 {
4336   snat_main_per_thread_data_t *tsm;
4337   clib_bihash_kv_8_8_t kv, value;
4338   ip4_header_t ip;
4339   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4340   snat_session_key_t key;
4341   snat_session_t *s;
4342   clib_bihash_8_8_t *t;
4343
4344   if (sm->endpoint_dependent)
4345     return VNET_API_ERROR_UNSUPPORTED;
4346
4347   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4348   if (sm->num_workers > 1)
4349     tsm =
4350       vec_elt_at_index (sm->per_thread_data,
4351                         sm->worker_in2out_cb (&ip, fib_index, 0));
4352   else
4353     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4354
4355   key.addr.as_u32 = addr->as_u32;
4356   key.port = clib_host_to_net_u16 (port);
4357   key.protocol = proto;
4358   key.fib_index = fib_index;
4359   kv.key = key.as_u64;
4360   t = is_in ? &tsm->in2out : &tsm->out2in;
4361   if (!clib_bihash_search_8_8 (t, &kv, &value))
4362     {
4363       if (pool_is_free_index (tsm->sessions, value.value))
4364         return VNET_API_ERROR_UNSPECIFIED;
4365
4366       s = pool_elt_at_index (tsm->sessions, value.value);
4367       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4368       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4369       return 0;
4370     }
4371
4372   return VNET_API_ERROR_NO_SUCH_ENTRY;
4373 }
4374
4375 int
4376 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4377                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4378                       u32 vrf_id, int is_in)
4379 {
4380   ip4_header_t ip;
4381   clib_bihash_16_8_t *t;
4382   nat_ed_ses_key_t key;
4383   clib_bihash_kv_16_8_t kv, value;
4384   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4385   snat_session_t *s;
4386   snat_main_per_thread_data_t *tsm;
4387
4388   if (!sm->endpoint_dependent)
4389     return VNET_API_ERROR_FEATURE_DISABLED;
4390
4391   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4392   if (sm->num_workers > 1)
4393     tsm =
4394       vec_elt_at_index (sm->per_thread_data,
4395                         sm->worker_in2out_cb (&ip, fib_index, 0));
4396   else
4397     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4398
4399   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4400   key.l_addr.as_u32 = addr->as_u32;
4401   key.r_addr.as_u32 = eh_addr->as_u32;
4402   key.l_port = clib_host_to_net_u16 (port);
4403   key.r_port = clib_host_to_net_u16 (eh_port);
4404   key.proto = proto;
4405   key.fib_index = fib_index;
4406   kv.key[0] = key.as_u64[0];
4407   kv.key[1] = key.as_u64[1];
4408   if (clib_bihash_search_16_8 (t, &kv, &value))
4409     return VNET_API_ERROR_NO_SUCH_ENTRY;
4410
4411   if (pool_is_free_index (tsm->sessions, value.value))
4412     return VNET_API_ERROR_UNSPECIFIED;
4413   s = pool_elt_at_index (tsm->sessions, value.value);
4414   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4415   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4416   return 0;
4417 }
4418
4419 void
4420 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4421 {
4422   snat_main_t *sm = &snat_main;
4423
4424   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4425   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4426   sm->psid = psid;
4427   sm->psid_offset = psid_offset;
4428   sm->psid_length = psid_length;
4429 }
4430
4431 void
4432 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4433 {
4434   snat_main_t *sm = &snat_main;
4435
4436   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4437   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4438   sm->start_port = start_port;
4439   sm->end_port = end_port;
4440 }
4441
4442 void
4443 nat_set_alloc_addr_and_port_default (void)
4444 {
4445   snat_main_t *sm = &snat_main;
4446
4447   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4448   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4449 }
4450
4451 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4452                                  vlib_node_runtime_t * node,
4453                                  vlib_frame_t * frame)
4454 {
4455   return 0;
4456 }
4457
4458 /* *INDENT-OFF* */
4459 VLIB_REGISTER_NODE (nat_default_node) = {
4460   .name = "nat-default",
4461   .vector_size = sizeof (u32),
4462   .format_trace = 0,
4463   .type = VLIB_NODE_TYPE_INTERNAL,
4464   .n_errors = 0,
4465   .n_next_nodes = NAT_N_NEXT,
4466   .next_nodes = {
4467     [NAT_NEXT_DROP] = "error-drop",
4468     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4469     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4470     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4471     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4472     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4473     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4474     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4475     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4476   },
4477 };
4478 /* *INDENT-ON* */
4479
4480 /*
4481  * fd.io coding-style-patch-verification: ON
4482  *
4483  * Local Variables:
4484  * eval: (c-set-style "gnu")
4485  * End:
4486  */