src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36 #include <vppinfra/bihash_16_8.h>
  37
  38 #include <vpp/app/version.h>
  39
  40 snat_main_t snat_main;
  41
  42 fib_source_t nat_fib_src_hi;
  43 fib_source_t nat_fib_src_low;
  44
  45 /* *INDENT-OFF* */
  46 /* Hook up input features */
  47 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  48   .arc_name = "ip4-unicast",
  49   .node_name = "nat-pre-in2out",
  50   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  51                                "ip4-sv-reassembly-feature"),
  52 };
  53 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  54   .arc_name = "ip4-unicast",
  55   .node_name = "nat-pre-out2in",
  56   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  57                                "ip4-dhcp-client-detect",
  58                                "ip4-sv-reassembly-feature"),
  59 };
  60 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  61   .arc_name = "ip4-unicast",
  62   .node_name = "nat44-in2out-worker-handoff",
  63   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  64 };
  65 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  66   .arc_name = "ip4-unicast",
  67   .node_name = "nat44-out2in-worker-handoff",
  68   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  69                                "ip4-dhcp-client-detect"),
  70 };
  71 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  72   .arc_name = "ip4-unicast",
  73   .node_name = "nat44-in2out",
  74   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  75 };
  76 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  77   .arc_name = "ip4-unicast",
  78   .node_name = "nat44-out2in",
  79   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  80                                "ip4-dhcp-client-detect"),
  81 };
  82 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  83   .arc_name = "ip4-unicast",
  84   .node_name = "nat44-classify",
  85   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  86 };
  87 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  88   .arc_name = "ip4-unicast",
  89   .node_name = "nat44-det-in2out",
  90   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-det-out2in",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  96                                "ip4-dhcp-client-detect"),
  97 };
  98 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-det-classify",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 102 };
 103 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-ed-in2out",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-ed-out2in",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 112                                "ip4-dhcp-client-detect"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-ed-classify",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 118 };
 119 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-handoff-classify",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 123 };
 124 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-in2out-fast",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 128 };
 129 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 130   .arc_name = "ip4-unicast",
 131   .node_name = "nat44-out2in-fast",
 132   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 133                                "ip4-dhcp-client-detect"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 136   .arc_name = "ip4-unicast",
 137   .node_name = "nat44-hairpin-dst",
 138   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 141   .arc_name = "ip4-unicast",
 142   .node_name = "nat44-ed-hairpin-dst",
 143   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 144 };
 145
 146 /* Hook up output features */
 147 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 148   .arc_name = "ip4-output",
 149   .node_name = "nat44-in2out-output",
 150   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 151 };
 152 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 153   .arc_name = "ip4-output",
 154   .node_name = "nat44-in2out-output-worker-handoff",
 155   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 156 };
 157 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 158   .arc_name = "ip4-output",
 159   .node_name = "nat44-hairpin-src",
 160   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 161 };
 162 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 163   .arc_name = "ip4-output",
 164   .node_name = "nat44-ed-in2out-output",
 165   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 166   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 167 };
 168 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 169   .arc_name = "ip4-output",
 170   .node_name = "nat44-ed-hairpin-src",
 171   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 172   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 173 };
 174
 175 /* Hook up ip4-local features */
 176 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 177 {
 178   .arc_name = "ip4-local",
 179   .node_name = "nat44-hairpinning",
 180   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 181 };
 182 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 183 {
 184   .arc_name = "ip4-local",
 185   .node_name = "nat44-ed-hairpinning",
 186   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 187 };
 188
 189
 190 VLIB_PLUGIN_REGISTER () = {
 191     .version = VPP_BUILD_VER,
 192     .description = "Network Address Translation (NAT)",
 193 };
 194 /* *INDENT-ON* */
 195
 196 void
 197 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 198                        u8 is_ha)
 199 {
 200   snat_session_key_t key;
 201   clib_bihash_kv_8_8_t kv;
 202   nat_ed_ses_key_t ed_key;
 203   clib_bihash_kv_16_8_t ed_kv;
 204   snat_main_per_thread_data_t *tsm =
 205     vec_elt_at_index (sm->per_thread_data, thread_index);
 206
 207   if (is_fwd_bypass_session (s))
 208     {
 209       if (snat_is_unk_proto_session (s))
 210         {
 211           ed_key.proto = s->in2out.port;
 212           ed_key.r_port = 0;
 213           ed_key.l_port = 0;
 214         }
 215       else
 216         {
 217           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 218           ed_key.l_port = s->in2out.port;
 219           ed_key.r_port = s->ext_host_port;
 220         }
 221       ed_key.l_addr = s->in2out.addr;
 222       ed_key.r_addr = s->ext_host_addr;
 223       ed_key.fib_index = 0;
 224       ed_kv.key[0] = ed_key.as_u64[0];
 225       ed_kv.key[1] = ed_key.as_u64[1];
 226       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 227         nat_elog_warn ("in2out_ed key del failed");
 228       return;
 229     }
 230
 231   /* session lookup tables */
 232   if (is_ed_session (s))
 233     {
 234       if (is_affinity_sessions (s))
 235         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 236                              s->in2out.protocol, s->out2in.port);
 237       ed_key.l_addr = s->out2in.addr;
 238       ed_key.r_addr = s->ext_host_addr;
 239       ed_key.fib_index = s->out2in.fib_index;
 240       if (snat_is_unk_proto_session (s))
 241         {
 242           ed_key.proto = s->in2out.port;
 243           ed_key.r_port = 0;
 244           ed_key.l_port = 0;
 245         }
 246       else
 247         {
 248           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 249           ed_key.l_port = s->out2in.port;
 250           ed_key.r_port = s->ext_host_port;
 251         }
 252       ed_kv.key[0] = ed_key.as_u64[0];
 253       ed_kv.key[1] = ed_key.as_u64[1];
 254       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 255         nat_elog_warn ("out2in_ed key del failed");
 256       ed_key.l_addr = s->in2out.addr;
 257       ed_key.fib_index = s->in2out.fib_index;
 258       if (!snat_is_unk_proto_session (s))
 259         ed_key.l_port = s->in2out.port;
 260       if (is_twice_nat_session (s))
 261         {
 262           ed_key.r_addr = s->ext_host_nat_addr;
 263           ed_key.r_port = s->ext_host_nat_port;
 264         }
 265       ed_kv.key[0] = ed_key.as_u64[0];
 266       ed_kv.key[1] = ed_key.as_u64[1];
 267       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 268         nat_elog_warn ("in2out_ed key del failed");
 269
 270       if (!is_ha)
 271         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 272                                &s->in2out.addr, s->in2out.port,
 273                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 274                                &s->out2in.addr, s->out2in.port,
 275                                &s->ext_host_addr, s->ext_host_port,
 276                                s->in2out.protocol, is_twice_nat_session (s));
 277     }
 278   else
 279     {
 280       kv.key = s->in2out.as_u64;
 281       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 282         nat_elog_warn ("in2out key del failed");
 283       kv.key = s->out2in.as_u64;
 284       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 285         nat_elog_warn ("out2in key del failed");
 286
 287       if (!is_ha)
 288         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 289                                  &s->in2out.addr, s->in2out.port,
 290                                  &s->out2in.addr, s->out2in.port,
 291                                  s->in2out.protocol);
 292     }
 293
 294   if (snat_is_unk_proto_session (s))
 295     return;
 296
 297   if (!is_ha)
 298     {
 299       /* log NAT event */
 300       snat_ipfix_logging_nat44_ses_delete (thread_index,
 301                                            s->in2out.addr.as_u32,
 302                                            s->out2in.addr.as_u32,
 303                                            s->in2out.protocol,
 304                                            s->in2out.port,
 305                                            s->out2in.port,
 306                                            s->in2out.fib_index);
 307
 308       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 309                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 310                    thread_index);
 311     }
 312
 313   /* Twice NAT address and port for external host */
 314   if (is_twice_nat_session (s))
 315     {
 316       key.protocol = s->in2out.protocol;
 317       key.port = s->ext_host_nat_port;
 318       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 319       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 320                                           thread_index, &key);
 321     }
 322
 323   if (snat_is_session_static (s))
 324     return;
 325
 326   snat_free_outside_address_and_port (sm->addresses, thread_index,
 327                                       &s->out2in);
 328 }
 329
 330 void
 331 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 332                          u32 thread_index, u8 is_ha)
 333 {
 334   snat_session_key_t key;
 335   nat_ed_ses_key_t ed_key;
 336   clib_bihash_kv_16_8_t ed_kv;
 337   snat_main_per_thread_data_t *tsm =
 338     vec_elt_at_index (sm->per_thread_data, thread_index);
 339
 340   if (is_fwd_bypass_session (s))
 341     {
 342       if (snat_is_unk_proto_session (s))
 343         {
 344           ed_key.proto = s->in2out.port;
 345           ed_key.r_port = 0;
 346           ed_key.l_port = 0;
 347         }
 348       else
 349         {
 350           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 351           ed_key.l_port = s->in2out.port;
 352           ed_key.r_port = s->ext_host_port;
 353         }
 354
 355       ed_key.l_addr = s->in2out.addr;
 356       ed_key.r_addr = s->ext_host_addr;
 357       ed_key.fib_index = 0;
 358       ed_kv.key[0] = ed_key.as_u64[0];
 359       ed_kv.key[1] = ed_key.as_u64[1];
 360
 361       if (PREDICT_FALSE
 362           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 363         nat_elog_warn ("in2out_ed key del failed");
 364       return;
 365     }
 366
 367   /* session lookup tables */
 368   if (is_affinity_sessions (s))
 369     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 370                          s->in2out.protocol, s->out2in.port);
 371   ed_key.l_addr = s->out2in.addr;
 372   ed_key.r_addr = s->ext_host_addr;
 373   ed_key.fib_index = s->out2in.fib_index;
 374   if (snat_is_unk_proto_session (s))
 375     {
 376       ed_key.proto = s->in2out.port;
 377       ed_key.r_port = 0;
 378       ed_key.l_port = 0;
 379     }
 380   else
 381     {
 382       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 383       ed_key.l_port = s->out2in.port;
 384       ed_key.r_port = s->ext_host_port;
 385     }
 386   ed_kv.key[0] = ed_key.as_u64[0];
 387   ed_kv.key[1] = ed_key.as_u64[1];
 388
 389   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0)))
 390     nat_elog_warn ("out2in_ed key del failed");
 391
 392   ed_key.l_addr = s->in2out.addr;
 393   ed_key.fib_index = s->in2out.fib_index;
 394
 395   if (!snat_is_unk_proto_session (s))
 396     ed_key.l_port = s->in2out.port;
 397
 398   if (is_twice_nat_session (s))
 399     {
 400       ed_key.r_addr = s->ext_host_nat_addr;
 401       ed_key.r_port = s->ext_host_nat_port;
 402     }
 403
 404   ed_kv.key[0] = ed_key.as_u64[0];
 405   ed_kv.key[1] = ed_key.as_u64[1];
 406
 407   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 408     nat_elog_warn ("in2out_ed key del failed");
 409
 410   if (!is_ha)
 411     {
 412       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 413                              &s->in2out.addr, s->in2out.port,
 414                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 415                              &s->out2in.addr, s->out2in.port,
 416                              &s->ext_host_addr, s->ext_host_port,
 417                              s->in2out.protocol, is_twice_nat_session (s));
 418     }
 419
 420   if (snat_is_unk_proto_session (s))
 421     return;
 422
 423   // is this correct ?
 424   if (!is_ha)
 425     {
 426       snat_ipfix_logging_nat44_ses_delete (thread_index,
 427                                            s->in2out.addr.as_u32,
 428                                            s->out2in.addr.as_u32,
 429                                            s->in2out.protocol,
 430                                            s->in2out.port,
 431                                            s->out2in.port,
 432                                            s->in2out.fib_index);
 433       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 434                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 435                    thread_index);
 436     }
 437
 438   /* Twice NAT address and port for external host */
 439   if (is_twice_nat_session (s))
 440     {
 441       key.protocol = s->in2out.protocol;
 442       key.port = s->ext_host_nat_port;
 443       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 444       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 445                                           thread_index, &key);
 446     }
 447
 448   if (snat_is_session_static (s))
 449     return;
 450
 451   // should be called for every dynamic session
 452   snat_free_outside_address_and_port (sm->addresses, thread_index,
 453                                       &s->out2in);
 454 }
 455
 456
 457 snat_user_t *
 458 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 459                         u32 thread_index)
 460 {
 461   snat_user_t *u = 0;
 462   snat_user_key_t user_key;
 463   clib_bihash_kv_8_8_t kv, value;
 464   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 465   dlist_elt_t *per_user_list_head_elt;
 466
 467   user_key.addr.as_u32 = addr->as_u32;
 468   user_key.fib_index = fib_index;
 469   kv.key = user_key.as_u64;
 470
 471   /* Ever heard of the "user" = src ip4 address before? */
 472   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 473     {
 474       /* no, make a new one */
 475       pool_get (tsm->users, u);
 476       clib_memset (u, 0, sizeof (*u));
 477
 478       u->addr.as_u32 = addr->as_u32;
 479       u->fib_index = fib_index;
 480
 481       pool_get (tsm->list_pool, per_user_list_head_elt);
 482
 483       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 484         tsm->list_pool;
 485
 486       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 487
 488       kv.value = u - tsm->users;
 489
 490       /* add user */
 491       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 492         {
 493           nat_elog_warn ("user_hash key add failed");
 494           nat44_delete_user_with_no_session (sm, u, thread_index);
 495           return NULL;
 496         }
 497
 498       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 499                                pool_elts (tsm->users));
 500     }
 501   else
 502     {
 503       u = pool_elt_at_index (tsm->users, value.value);
 504     }
 505
 506   return u;
 507 }
 508
 509 snat_session_t *
 510 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 511                               u32 thread_index, f64 now)
 512 {
 513   snat_session_t *s;
 514   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 515   u32 oldest_per_user_translation_list_index, session_index;
 516   dlist_elt_t *oldest_per_user_translation_list_elt;
 517   dlist_elt_t *per_user_translation_list_elt;
 518
 519   /* Over quota? Recycle the least recently used translation */
 520   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 521     {
 522       oldest_per_user_translation_list_index =
 523         clib_dlist_remove_head (tsm->list_pool,
 524                                 u->sessions_per_user_list_head_index);
 525
 526       ASSERT (oldest_per_user_translation_list_index != ~0);
 527
 528       /* Add it back to the end of the LRU list */
 529       clib_dlist_addtail (tsm->list_pool,
 530                           u->sessions_per_user_list_head_index,
 531                           oldest_per_user_translation_list_index);
 532       /* Get the list element */
 533       oldest_per_user_translation_list_elt =
 534         pool_elt_at_index (tsm->list_pool,
 535                            oldest_per_user_translation_list_index);
 536
 537       /* Get the session index from the list element */
 538       session_index = oldest_per_user_translation_list_elt->value;
 539
 540       /* Get the session */
 541       s = pool_elt_at_index (tsm->sessions, session_index);
 542       nat_free_session_data (sm, s, thread_index, 0);
 543       if (snat_is_session_static (s))
 544         u->nstaticsessions--;
 545       else
 546         u->nsessions--;
 547       s->flags = 0;
 548       s->total_bytes = 0;
 549       s->total_pkts = 0;
 550       s->state = 0;
 551       s->ext_host_addr.as_u32 = 0;
 552       s->ext_host_port = 0;
 553       s->ext_host_nat_addr.as_u32 = 0;
 554       s->ext_host_nat_port = 0;
 555     }
 556   else
 557     {
 558       pool_get (tsm->sessions, s);
 559       clib_memset (s, 0, sizeof (*s));
 560
 561       /* Create list elts */
 562       pool_get (tsm->list_pool, per_user_translation_list_elt);
 563       clib_dlist_init (tsm->list_pool,
 564                        per_user_translation_list_elt - tsm->list_pool);
 565
 566       per_user_translation_list_elt->value = s - tsm->sessions;
 567       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 568       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 569
 570       clib_dlist_addtail (tsm->list_pool,
 571                           s->per_user_list_head_index,
 572                           per_user_translation_list_elt - tsm->list_pool);
 573
 574       dlist_elt_t *global_lru_list_elt;
 575       pool_get (tsm->global_lru_pool, global_lru_list_elt);
 576       global_lru_list_elt->value = s - tsm->sessions;
 577       s->global_lru_index = global_lru_list_elt - tsm->global_lru_pool;
 578       clib_dlist_addtail (tsm->global_lru_pool, tsm->global_lru_head_index,
 579                           s->global_lru_index);
 580       s->last_lru_update = now;
 581
 582       s->user_index = u - tsm->users;
 583       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 584                                pool_elts (tsm->sessions));
 585     }
 586
 587   s->ha_last_refreshed = now;
 588
 589   return s;
 590 }
 591
 592 int
 593 nat_global_lru_free_one (snat_main_t * sm, int thread_index, f64 now)
 594 {
 595   snat_session_t *s = NULL;
 596   dlist_elt_t *oldest_elt;
 597   u64 sess_timeout_time;
 598   u32 oldest_index;
 599   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 600   oldest_index = clib_dlist_remove_head (tsm->global_lru_pool,
 601                                          tsm->global_lru_head_index);
 602   if (~0 != oldest_index)
 603     {
 604       oldest_elt = pool_elt_at_index (tsm->global_lru_pool, oldest_index);
 605       s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 606
 607       sess_timeout_time =
 608         s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 609       if (now >= sess_timeout_time
 610           || (s->tcp_close_timestamp && now >= s->tcp_close_timestamp))
 611         {
 612           nat_free_session_data (sm, s, thread_index, 0);
 613           nat44_ed_delete_session (sm, s, thread_index, 0);
 614           return 1;
 615         }
 616       else
 617         {
 618           clib_dlist_addhead (tsm->global_lru_pool,
 619                               tsm->global_lru_head_index, oldest_index);
 620         }
 621     }
 622   return 0;
 623 }
 624
 625 snat_session_t *
 626 nat_ed_session_alloc (snat_main_t * sm, u32 thread_index, f64 now)
 627 {
 628   snat_session_t *s;
 629   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 630
 631   nat_global_lru_free_one (sm, thread_index, now);
 632
 633   pool_get (tsm->sessions, s);
 634   clib_memset (s, 0, sizeof (*s));
 635
 636   nat44_global_lru_insert (tsm, s, now);
 637
 638   s->ha_last_refreshed = now;
 639   vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 640                            pool_elts (tsm->sessions));
 641   return s;
 642 }
 643
 644 void
 645 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 646                           int is_add)
 647 {
 648   fib_prefix_t prefix = {
 649     .fp_len = p_len,
 650     .fp_proto = FIB_PROTOCOL_IP4,
 651     .fp_addr = {
 652                 .ip4.as_u32 = addr->as_u32,
 653                 },
 654   };
 655   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 656
 657   if (is_add)
 658     fib_table_entry_update_one_path (fib_index,
 659                                      &prefix,
 660                                      nat_fib_src_low,
 661                                      (FIB_ENTRY_FLAG_CONNECTED |
 662                                       FIB_ENTRY_FLAG_LOCAL |
 663                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 664                                      DPO_PROTO_IP4,
 665                                      NULL,
 666                                      sw_if_index,
 667                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 668   else
 669     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 670 }
 671
 672 int
 673 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 674                   u8 twice_nat)
 675 {
 676   snat_address_t *ap;
 677   snat_interface_t *i;
 678   vlib_thread_main_t *tm = vlib_get_thread_main ();
 679
 680   if (twice_nat && !sm->endpoint_dependent)
 681     return VNET_API_ERROR_FEATURE_DISABLED;
 682
 683   /* Check if address already exists */
 684   /* *INDENT-OFF* */
 685   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 686     {
 687       if (ap->addr.as_u32 == addr->as_u32)
 688         return VNET_API_ERROR_VALUE_EXIST;
 689     }
 690   /* *INDENT-ON* */
 691
 692   if (twice_nat)
 693     vec_add2 (sm->twice_nat_addresses, ap, 1);
 694   else
 695     vec_add2 (sm->addresses, ap, 1);
 696
 697   ap->addr = *addr;
 698   if (vrf_id != ~0)
 699     ap->fib_index =
 700       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 701                                          nat_fib_src_low);
 702   else
 703     ap->fib_index = ~0;
 704 #define _(N, i, n, s) \
 705   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 706   ap->busy_##n##_ports = 0; \
 707   ap->busy_##n##_ports_per_thread = 0;\
 708   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 709   foreach_snat_protocol
 710 #undef _
 711     if (twice_nat)
 712     return 0;
 713
 714   /* Add external address to FIB */
 715   /* *INDENT-OFF* */
 716   pool_foreach (i, sm->interfaces,
 717   ({
 718     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 719       continue;
 720
 721     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 722     break;
 723   }));
 724   pool_foreach (i, sm->output_feature_interfaces,
 725   ({
 726     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 727       continue;
 728
 729     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 730     break;
 731   }));
 732   /* *INDENT-ON* */
 733
 734   return 0;
 735 }
 736
 737 static int
 738 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 739 {
 740   snat_static_mapping_t *m;
 741   /* *INDENT-OFF* */
 742   pool_foreach (m, sm->static_mappings,
 743   ({
 744       if (is_addr_only_static_mapping (m) ||
 745           is_out2in_only_static_mapping (m) ||
 746           is_identity_static_mapping (m))
 747         continue;
 748       if (m->external_addr.as_u32 == addr.as_u32)
 749         return 1;
 750   }));
 751   /* *INDENT-ON* */
 752
 753   return 0;
 754 }
 755
 756 static void
 757 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 758                                        ip4_address_t l_addr,
 759                                        u16 l_port,
 760                                        u32 sw_if_index,
 761                                        u16 e_port,
 762                                        u32 vrf_id,
 763                                        snat_protocol_t proto,
 764                                        int addr_only, int is_add, u8 * tag,
 765                                        int twice_nat, int out2in_only,
 766                                        int identity_nat)
 767 {
 768   snat_static_map_resolve_t *rp;
 769
 770   vec_add2 (sm->to_resolve, rp, 1);
 771   rp->l_addr.as_u32 = l_addr.as_u32;
 772   rp->l_port = l_port;
 773   rp->sw_if_index = sw_if_index;
 774   rp->e_port = e_port;
 775   rp->vrf_id = vrf_id;
 776   rp->proto = proto;
 777   rp->addr_only = addr_only;
 778   rp->is_add = is_add;
 779   rp->twice_nat = twice_nat;
 780   rp->out2in_only = out2in_only;
 781   rp->identity_nat = identity_nat;
 782   rp->tag = vec_dup (tag);
 783 }
 784
 785 static u32
 786 get_thread_idx_by_port (u16 e_port)
 787 {
 788   snat_main_t *sm = &snat_main;
 789   u32 thread_idx = sm->num_workers;
 790   if (sm->num_workers > 1)
 791     {
 792       thread_idx =
 793         sm->first_worker_index +
 794         sm->workers[(e_port - 1024) / sm->port_per_thread];
 795     }
 796   return thread_idx;
 797 }
 798
 799 void
 800 snat_static_mapping_del_sessions (snat_main_t * sm,
 801                                   snat_main_per_thread_data_t * tsm,
 802                                   snat_user_key_t u_key, int addr_only,
 803                                   ip4_address_t e_addr, u16 e_port)
 804 {
 805   clib_bihash_kv_8_8_t kv, value;
 806   kv.key = u_key.as_u64;
 807   u64 user_index;
 808   dlist_elt_t *head, *elt;
 809   snat_user_t *u;
 810   snat_session_t *s;
 811   u32 elt_index, head_index, ses_index;
 812   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 813     {
 814       user_index = value.value;
 815       u = pool_elt_at_index (tsm->users, user_index);
 816       if (u->nstaticsessions)
 817         {
 818           head_index = u->sessions_per_user_list_head_index;
 819           head = pool_elt_at_index (tsm->list_pool, head_index);
 820           elt_index = head->next;
 821           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 822           ses_index = elt->value;
 823           while (ses_index != ~0)
 824             {
 825               s = pool_elt_at_index (tsm->sessions, ses_index);
 826               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 827               ses_index = elt->value;
 828
 829               if (!addr_only)
 830                 {
 831                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 832                       (clib_net_to_host_u16 (s->out2in.port) != e_port))
 833                     continue;
 834                 }
 835
 836               if (is_lb_session (s))
 837                 continue;
 838
 839               if (!snat_is_session_static (s))
 840                 continue;
 841
 842               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 843               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 844
 845               if (!addr_only)
 846                 break;
 847             }
 848         }
 849     }
 850 }
 851
 852 void
 853 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 854                                      snat_main_per_thread_data_t * tsm,
 855                                      ip4_address_t l_addr,
 856                                      u16 l_port,
 857                                      u8 protocol,
 858                                      u32 fib_index, int addr_only,
 859                                      ip4_address_t e_addr, u16 e_port)
 860 {
 861   snat_session_t *s;
 862   u32 *indexes_to_free = NULL;
 863   /* *INDENT-OFF* */
 864   pool_foreach (s, tsm->sessions, {
 865     if (s->in2out.fib_index != fib_index ||
 866         s->in2out.addr.as_u32 != l_addr.as_u32)
 867       {
 868         continue;
 869       }
 870     if (!addr_only)
 871       {
 872         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 873             (clib_net_to_host_u16 (s->out2in.port) != e_port) ||
 874             clib_net_to_host_u16 (s->in2out.port) != l_port ||
 875             s->in2out.protocol != protocol)
 876           continue;
 877       }
 878
 879     if (is_lb_session (s))
 880       continue;
 881     if (!snat_is_session_static (s))
 882       continue;
 883     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 884     vec_add1 (indexes_to_free, s - tsm->sessions);
 885     if (!addr_only)
 886       break;
 887   });
 888   /* *INDENT-ON* */
 889   u32 *ses_index;
 890   vec_foreach (ses_index, indexes_to_free)
 891   {
 892     s = pool_elt_at_index (tsm->sessions, *ses_index);
 893     nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
 894   }
 895   vec_free (indexes_to_free);
 896 }
 897
 898 int
 899 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 900                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 901                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 902                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 903                          u8 identity_nat)
 904 {
 905   snat_main_t *sm = &snat_main;
 906   snat_static_mapping_t *m;
 907   snat_session_key_t m_key;
 908   clib_bihash_kv_8_8_t kv, value;
 909   snat_address_t *a = 0;
 910   u32 fib_index = ~0;
 911   snat_interface_t *interface;
 912   int i;
 913   snat_main_per_thread_data_t *tsm;
 914   snat_user_key_t u_key;
 915   snat_user_t *u;
 916   dlist_elt_t *head, *elt;
 917   u32 elt_index, head_index;
 918   u32 ses_index;
 919   u64 user_index;
 920   snat_session_t *s;
 921   snat_static_map_resolve_t *rp, *rp_match = 0;
 922   nat44_lb_addr_port_t *local;
 923   u32 find = ~0;
 924
 925   if (!sm->endpoint_dependent)
 926     {
 927       if (twice_nat || out2in_only)
 928         return VNET_API_ERROR_FEATURE_DISABLED;
 929     }
 930
 931   /* If the external address is a specific interface address */
 932   if (sw_if_index != ~0)
 933     {
 934       ip4_address_t *first_int_addr;
 935
 936       for (i = 0; i < vec_len (sm->to_resolve); i++)
 937         {
 938           rp = sm->to_resolve + i;
 939           if (rp->sw_if_index != sw_if_index ||
 940               rp->l_addr.as_u32 != l_addr.as_u32 ||
 941               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 942             continue;
 943
 944           if (!addr_only)
 945             {
 946               if ((rp->l_port != l_port && rp->e_port != e_port)
 947                   || rp->proto != proto)
 948                 continue;
 949             }
 950
 951           rp_match = rp;
 952           break;
 953         }
 954
 955       /* Might be already set... */
 956       first_int_addr = ip4_interface_first_address
 957         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 958
 959       if (is_add)
 960         {
 961           if (rp_match)
 962             return VNET_API_ERROR_VALUE_EXIST;
 963
 964           snat_add_static_mapping_when_resolved
 965             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 966              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 967
 968           /* DHCP resolution required? */
 969           if (first_int_addr == 0)
 970             {
 971               return 0;
 972             }
 973           else
 974             {
 975               e_addr.as_u32 = first_int_addr->as_u32;
 976               /* Identity mapping? */
 977               if (l_addr.as_u32 == 0)
 978                 l_addr.as_u32 = e_addr.as_u32;
 979             }
 980         }
 981       else
 982         {
 983           if (!rp_match)
 984             return VNET_API_ERROR_NO_SUCH_ENTRY;
 985
 986           vec_del1 (sm->to_resolve, i);
 987
 988           if (first_int_addr)
 989             {
 990               e_addr.as_u32 = first_int_addr->as_u32;
 991               /* Identity mapping? */
 992               if (l_addr.as_u32 == 0)
 993                 l_addr.as_u32 = e_addr.as_u32;
 994             }
 995           else
 996             return 0;
 997         }
 998     }
 999
1000   m_key.addr = e_addr;
1001   m_key.port = addr_only ? 0 : e_port;
1002   m_key.protocol = addr_only ? 0 : proto;
1003   m_key.fib_index = 0;
1004   kv.key = m_key.as_u64;
1005   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1006     m = 0;
1007   else
1008     m = pool_elt_at_index (sm->static_mappings, value.value);
1009
1010   if (is_add)
1011     {
1012       if (m)
1013         {
1014           if (is_identity_static_mapping (m))
1015             {
1016               /* *INDENT-OFF* */
1017               pool_foreach (local, m->locals,
1018               ({
1019                 if (local->vrf_id == vrf_id)
1020                   return VNET_API_ERROR_VALUE_EXIST;
1021               }));
1022               /* *INDENT-ON* */
1023               pool_get (m->locals, local);
1024               local->vrf_id = vrf_id;
1025               local->fib_index =
1026                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1027                                                    nat_fib_src_low);
1028               m_key.addr = m->local_addr;
1029               m_key.port = m->local_port;
1030               m_key.protocol = m->proto;
1031               m_key.fib_index = local->fib_index;
1032               kv.key = m_key.as_u64;
1033               kv.value = m - sm->static_mappings;
1034               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1035               return 0;
1036             }
1037           else
1038             return VNET_API_ERROR_VALUE_EXIST;
1039         }
1040
1041       if (twice_nat && addr_only)
1042         return VNET_API_ERROR_UNSUPPORTED;
1043
1044       /* Convert VRF id to FIB index */
1045       if (vrf_id != ~0)
1046         fib_index =
1047           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1048                                              nat_fib_src_low);
1049       /* If not specified use inside VRF id from SNAT plugin startup config */
1050       else
1051         {
1052           fib_index = sm->inside_fib_index;
1053           vrf_id = sm->inside_vrf_id;
1054           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1055         }
1056
1057       if (!(out2in_only || identity_nat))
1058         {
1059           m_key.addr = l_addr;
1060           m_key.port = addr_only ? 0 : l_port;
1061           m_key.protocol = addr_only ? 0 : proto;
1062           m_key.fib_index = fib_index;
1063           kv.key = m_key.as_u64;
1064           if (!clib_bihash_search_8_8
1065               (&sm->static_mapping_by_local, &kv, &value))
1066             return VNET_API_ERROR_VALUE_EXIST;
1067         }
1068
1069       /* Find external address in allocated addresses and reserve port for
1070          address and port pair mapping when dynamic translations enabled */
1071       if (!(addr_only || sm->static_mapping_only || out2in_only))
1072         {
1073           for (i = 0; i < vec_len (sm->addresses); i++)
1074             {
1075               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1076                 {
1077                   a = sm->addresses + i;
1078                   /* External port must be unused */
1079                   switch (proto)
1080                     {
1081 #define _(N, j, n, s) \
1082                     case SNAT_PROTOCOL_##N: \
1083                       if (a->busy_##n##_port_refcounts[e_port]) \
1084                         return VNET_API_ERROR_INVALID_VALUE; \
1085                       ++a->busy_##n##_port_refcounts[e_port]; \
1086                       if (e_port > 1024) \
1087                         { \
1088                           a->busy_##n##_ports++; \
1089                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1090                         } \
1091                       break;
1092                       foreach_snat_protocol
1093 #undef _
1094                     default:
1095                       nat_elog_info ("unknown protocol");
1096                       return VNET_API_ERROR_INVALID_VALUE_2;
1097                     }
1098                   break;
1099                 }
1100             }
1101           /* External address must be allocated */
1102           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1103             {
1104               if (sw_if_index != ~0)
1105                 {
1106                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1107                     {
1108                       rp = sm->to_resolve + i;
1109                       if (rp->addr_only)
1110                         continue;
1111                       if (rp->sw_if_index != sw_if_index &&
1112                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1113                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1114                           rp->e_port != e_port && rp->proto != proto)
1115                         continue;
1116
1117                       vec_del1 (sm->to_resolve, i);
1118                       break;
1119                     }
1120                 }
1121               return VNET_API_ERROR_NO_SUCH_ENTRY;
1122             }
1123         }
1124
1125       pool_get (sm->static_mappings, m);
1126       clib_memset (m, 0, sizeof (*m));
1127       m->tag = vec_dup (tag);
1128       m->local_addr = l_addr;
1129       m->external_addr = e_addr;
1130       m->twice_nat = twice_nat;
1131       if (out2in_only)
1132         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1133       if (addr_only)
1134         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1135       if (identity_nat)
1136         {
1137           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1138           pool_get (m->locals, local);
1139           local->vrf_id = vrf_id;
1140           local->fib_index = fib_index;
1141         }
1142       else
1143         {
1144           m->vrf_id = vrf_id;
1145           m->fib_index = fib_index;
1146         }
1147       if (!addr_only)
1148         {
1149           m->local_port = l_port;
1150           m->external_port = e_port;
1151           m->proto = proto;
1152         }
1153
1154       if (sm->num_workers > 1)
1155         {
1156           ip4_header_t ip = {
1157             .src_address = m->local_addr,
1158           };
1159           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1160           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1161         }
1162       else
1163         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1164
1165       m_key.addr = m->local_addr;
1166       m_key.port = m->local_port;
1167       m_key.protocol = m->proto;
1168       m_key.fib_index = fib_index;
1169       kv.key = m_key.as_u64;
1170       kv.value = m - sm->static_mappings;
1171       if (!out2in_only)
1172         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1173
1174       m_key.addr = m->external_addr;
1175       m_key.port = m->external_port;
1176       m_key.fib_index = 0;
1177       kv.key = m_key.as_u64;
1178       kv.value = m - sm->static_mappings;
1179       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1180
1181       /* Delete dynamic sessions matching local address (+ local port) */
1182       if (!(sm->static_mapping_only))
1183         {
1184           u_key.addr = m->local_addr;
1185           u_key.fib_index = m->fib_index;
1186           kv.key = u_key.as_u64;
1187           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1188             {
1189               user_index = value.value;
1190               u = pool_elt_at_index (tsm->users, user_index);
1191               if (u->nsessions)
1192                 {
1193                   head_index = u->sessions_per_user_list_head_index;
1194                   head = pool_elt_at_index (tsm->list_pool, head_index);
1195                   elt_index = head->next;
1196                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1197                   ses_index = elt->value;
1198                   while (ses_index != ~0)
1199                     {
1200                       s = pool_elt_at_index (tsm->sessions, ses_index);
1201                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1202                       ses_index = elt->value;
1203
1204                       if (snat_is_session_static (s))
1205                         continue;
1206
1207                       if (!addr_only
1208                           && (clib_net_to_host_u16 (s->in2out.port) !=
1209                               m->local_port))
1210                         continue;
1211
1212                       nat_free_session_data (sm, s,
1213                                              tsm - sm->per_thread_data, 0);
1214                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1215
1216                       if (!addr_only && !sm->endpoint_dependent)
1217                         break;
1218                     }
1219                 }
1220             }
1221         }
1222     }
1223   else
1224     {
1225       if (!m)
1226         {
1227           if (sw_if_index != ~0)
1228             return 0;
1229           else
1230             return VNET_API_ERROR_NO_SUCH_ENTRY;
1231         }
1232
1233       if (identity_nat)
1234         {
1235           if (vrf_id == ~0)
1236             vrf_id = sm->inside_vrf_id;
1237
1238           /* *INDENT-OFF* */
1239           pool_foreach (local, m->locals,
1240           ({
1241             if (local->vrf_id == vrf_id)
1242               find = local - m->locals;
1243           }));
1244           /* *INDENT-ON* */
1245           if (find == ~0)
1246             return VNET_API_ERROR_NO_SUCH_ENTRY;
1247
1248           local = pool_elt_at_index (m->locals, find);
1249           fib_index = local->fib_index;
1250           pool_put (m->locals, local);
1251         }
1252       else
1253         fib_index = m->fib_index;
1254
1255       /* Free external address port */
1256       if (!(addr_only || sm->static_mapping_only || out2in_only))
1257         {
1258           for (i = 0; i < vec_len (sm->addresses); i++)
1259             {
1260               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1261                 {
1262                   a = sm->addresses + i;
1263                   switch (proto)
1264                     {
1265 #define _(N, j, n, s) \
1266                     case SNAT_PROTOCOL_##N: \
1267                       --a->busy_##n##_port_refcounts[e_port]; \
1268                       if (e_port > 1024) \
1269                         { \
1270                           a->busy_##n##_ports--; \
1271                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1272                         } \
1273                       break;
1274                       foreach_snat_protocol
1275 #undef _
1276                     default:
1277                       nat_elog_info ("unknown protocol");
1278                       return VNET_API_ERROR_INVALID_VALUE_2;
1279                     }
1280                   break;
1281                 }
1282             }
1283         }
1284
1285       if (sm->num_workers > 1)
1286         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1287       else
1288         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1289
1290       m_key.addr = m->local_addr;
1291       m_key.port = m->local_port;
1292       m_key.protocol = m->proto;
1293       m_key.fib_index = fib_index;
1294       kv.key = m_key.as_u64;
1295       if (!out2in_only)
1296         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1297
1298       /* Delete session(s) for static mapping if exist */
1299       if (!(sm->static_mapping_only) ||
1300           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1301         {
1302           if (sm->endpoint_dependent)
1303             {
1304               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1305                                                    m->local_port, m->proto,
1306                                                    fib_index, addr_only,
1307                                                    e_addr, e_port);
1308             }
1309           else
1310             {
1311               u_key.addr = m->local_addr;
1312               u_key.fib_index = fib_index;
1313               kv.key = u_key.as_u64;
1314               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1315                                                 e_addr, e_port);
1316             }
1317         }
1318
1319       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1320       if (pool_elts (m->locals))
1321         return 0;
1322
1323       m_key.addr = m->external_addr;
1324       m_key.port = m->external_port;
1325       m_key.fib_index = 0;
1326       kv.key = m_key.as_u64;
1327       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1328
1329       vec_free (m->tag);
1330       vec_free (m->workers);
1331       /* Delete static mapping from pool */
1332       pool_put (sm->static_mappings, m);
1333     }
1334
1335   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1336     return 0;
1337
1338   /* Add/delete external address to FIB */
1339   /* *INDENT-OFF* */
1340   pool_foreach (interface, sm->interfaces,
1341   ({
1342     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1343       continue;
1344
1345     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1346     break;
1347   }));
1348   pool_foreach (interface, sm->output_feature_interfaces,
1349   ({
1350     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1351       continue;
1352
1353     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1354     break;
1355   }));
1356   /* *INDENT-ON* */
1357
1358   return 0;
1359 }
1360
1361 int
1362 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1363                                  snat_protocol_t proto,
1364                                  nat44_lb_addr_port_t * locals, u8 is_add,
1365                                  twice_nat_type_t twice_nat, u8 out2in_only,
1366                                  u8 * tag, u32 affinity)
1367 {
1368   snat_main_t *sm = &snat_main;
1369   snat_static_mapping_t *m;
1370   snat_session_key_t m_key;
1371   clib_bihash_kv_8_8_t kv, value;
1372   snat_address_t *a = 0;
1373   int i;
1374   nat44_lb_addr_port_t *local;
1375   snat_main_per_thread_data_t *tsm;
1376   snat_session_t *s;
1377   uword *bitmap = 0;
1378
1379   if (!sm->endpoint_dependent)
1380     return VNET_API_ERROR_FEATURE_DISABLED;
1381
1382   m_key.addr = e_addr;
1383   m_key.port = e_port;
1384   m_key.protocol = proto;
1385   m_key.fib_index = 0;
1386   kv.key = m_key.as_u64;
1387   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1388     m = 0;
1389   else
1390     m = pool_elt_at_index (sm->static_mappings, value.value);
1391
1392   if (is_add)
1393     {
1394       if (m)
1395         return VNET_API_ERROR_VALUE_EXIST;
1396
1397       if (vec_len (locals) < 2)
1398         return VNET_API_ERROR_INVALID_VALUE;
1399
1400       /* Find external address in allocated addresses and reserve port for
1401          address and port pair mapping when dynamic translations enabled */
1402       if (!(sm->static_mapping_only || out2in_only))
1403         {
1404           for (i = 0; i < vec_len (sm->addresses); i++)
1405             {
1406               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1407                 {
1408                   a = sm->addresses + i;
1409                   /* External port must be unused */
1410                   switch (proto)
1411                     {
1412 #define _(N, j, n, s) \
1413                     case SNAT_PROTOCOL_##N: \
1414                       if (a->busy_##n##_port_refcounts[e_port]) \
1415                         return VNET_API_ERROR_INVALID_VALUE; \
1416                       ++a->busy_##n##_port_refcounts[e_port]; \
1417                       if (e_port > 1024) \
1418                         { \
1419                           a->busy_##n##_ports++; \
1420                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1421                         } \
1422                       break;
1423                       foreach_snat_protocol
1424 #undef _
1425                     default:
1426                       nat_elog_info ("unknown protocol");
1427                       return VNET_API_ERROR_INVALID_VALUE_2;
1428                     }
1429                   break;
1430                 }
1431             }
1432           /* External address must be allocated */
1433           if (!a)
1434             return VNET_API_ERROR_NO_SUCH_ENTRY;
1435         }
1436
1437       pool_get (sm->static_mappings, m);
1438       clib_memset (m, 0, sizeof (*m));
1439       m->tag = vec_dup (tag);
1440       m->external_addr = e_addr;
1441       m->external_port = e_port;
1442       m->proto = proto;
1443       m->twice_nat = twice_nat;
1444       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1445       if (out2in_only)
1446         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1447       m->affinity = affinity;
1448
1449       if (affinity)
1450         m->affinity_per_service_list_head_index =
1451           nat_affinity_get_per_service_list_head_index ();
1452       else
1453         m->affinity_per_service_list_head_index = ~0;
1454
1455       m_key.addr = m->external_addr;
1456       m_key.port = m->external_port;
1457       m_key.protocol = m->proto;
1458       m_key.fib_index = 0;
1459       kv.key = m_key.as_u64;
1460       kv.value = m - sm->static_mappings;
1461       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1462         {
1463           nat_elog_err ("static_mapping_by_external key add failed");
1464           return VNET_API_ERROR_UNSPECIFIED;
1465         }
1466
1467       m_key.fib_index = m->fib_index;
1468       for (i = 0; i < vec_len (locals); i++)
1469         {
1470           locals[i].fib_index =
1471             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1472                                                locals[i].vrf_id,
1473                                                nat_fib_src_low);
1474           m_key.addr = locals[i].addr;
1475           m_key.fib_index = locals[i].fib_index;
1476           if (!out2in_only)
1477             {
1478               m_key.port = locals[i].port;
1479               kv.key = m_key.as_u64;
1480               kv.value = m - sm->static_mappings;
1481               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1482             }
1483           locals[i].prefix = (i == 0) ? locals[i].probability :
1484             (locals[i - 1].prefix + locals[i].probability);
1485           pool_get (m->locals, local);
1486           *local = locals[i];
1487           if (sm->num_workers > 1)
1488             {
1489               ip4_header_t ip = {
1490                 .src_address = locals[i].addr,
1491               };
1492               bitmap =
1493                 clib_bitmap_set (bitmap,
1494                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1495                                  1);
1496             }
1497         }
1498
1499       /* Assign workers */
1500       if (sm->num_workers > 1)
1501         {
1502           /* *INDENT-OFF* */
1503           clib_bitmap_foreach (i, bitmap,
1504             ({
1505                vec_add1(m->workers, i);
1506             }));
1507           /* *INDENT-ON* */
1508         }
1509     }
1510   else
1511     {
1512       if (!m)
1513         return VNET_API_ERROR_NO_SUCH_ENTRY;
1514
1515       if (!is_lb_static_mapping (m))
1516         return VNET_API_ERROR_INVALID_VALUE;
1517
1518       /* Free external address port */
1519       if (!(sm->static_mapping_only || out2in_only))
1520         {
1521           for (i = 0; i < vec_len (sm->addresses); i++)
1522             {
1523               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1524                 {
1525                   a = sm->addresses + i;
1526                   switch (proto)
1527                     {
1528 #define _(N, j, n, s) \
1529                     case SNAT_PROTOCOL_##N: \
1530                       --a->busy_##n##_port_refcounts[e_port]; \
1531                       if (e_port > 1024) \
1532                         { \
1533                           a->busy_##n##_ports--; \
1534                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1535                         } \
1536                       break;
1537                       foreach_snat_protocol
1538 #undef _
1539                     default:
1540                       nat_elog_info ("unknown protocol");
1541                       return VNET_API_ERROR_INVALID_VALUE_2;
1542                     }
1543                   break;
1544                 }
1545             }
1546         }
1547
1548       m_key.addr = m->external_addr;
1549       m_key.port = m->external_port;
1550       m_key.protocol = m->proto;
1551       m_key.fib_index = 0;
1552       kv.key = m_key.as_u64;
1553       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1554         {
1555           nat_elog_err ("static_mapping_by_external key del failed");
1556           return VNET_API_ERROR_UNSPECIFIED;
1557         }
1558
1559       /* *INDENT-OFF* */
1560       pool_foreach (local, m->locals,
1561       ({
1562           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1563                             nat_fib_src_low);
1564           m_key.addr = local->addr;
1565           if (!out2in_only)
1566             {
1567               m_key.port = local->port;
1568               m_key.fib_index = local->fib_index;
1569               kv.key = m_key.as_u64;
1570               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1571                 {
1572                   nat_elog_err ("static_mapping_by_local key del failed");
1573                   return VNET_API_ERROR_UNSPECIFIED;
1574                 }
1575             }
1576
1577           if (sm->num_workers > 1)
1578             {
1579               ip4_header_t ip = {
1580                 .src_address = local->addr,
1581               };
1582               tsm = vec_elt_at_index (sm->per_thread_data,
1583                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1584             }
1585           else
1586             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1587
1588           /* Delete sessions */
1589           pool_foreach (s, tsm->sessions, {
1590             if (!(is_lb_session (s)))
1591               continue;
1592
1593             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1594                 (clib_net_to_host_u16 (s->in2out.port) != local->port))
1595               continue;
1596
1597             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1598             nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
1599           });
1600       }));
1601       /* *INDENT-ON* */
1602       if (m->affinity)
1603         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1604       pool_free (m->locals);
1605       vec_free (m->tag);
1606       vec_free (m->workers);
1607
1608       pool_put (sm->static_mappings, m);
1609     }
1610
1611   return 0;
1612 }
1613
1614 int
1615 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1616                                        ip4_address_t l_addr, u16 l_port,
1617                                        snat_protocol_t proto, u32 vrf_id,
1618                                        u8 probability, u8 is_add)
1619 {
1620   snat_main_t *sm = &snat_main;
1621   snat_static_mapping_t *m = 0;
1622   snat_session_key_t m_key;
1623   clib_bihash_kv_8_8_t kv, value;
1624   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1625   snat_main_per_thread_data_t *tsm;
1626   snat_session_t *s;
1627   u32 *locals = 0;
1628   uword *bitmap = 0;
1629   int i;
1630
1631   if (!sm->endpoint_dependent)
1632     return VNET_API_ERROR_FEATURE_DISABLED;
1633
1634   m_key.addr = e_addr;
1635   m_key.port = e_port;
1636   m_key.protocol = proto;
1637   m_key.fib_index = 0;
1638   kv.key = m_key.as_u64;
1639   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1640     m = pool_elt_at_index (sm->static_mappings, value.value);
1641
1642   if (!m)
1643     return VNET_API_ERROR_NO_SUCH_ENTRY;
1644
1645   if (!is_lb_static_mapping (m))
1646     return VNET_API_ERROR_INVALID_VALUE;
1647
1648   /* *INDENT-OFF* */
1649   pool_foreach (local, m->locals,
1650   ({
1651     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1652         (local->vrf_id == vrf_id))
1653       {
1654         match_local = local;
1655         break;
1656       }
1657   }));
1658   /* *INDENT-ON* */
1659
1660   if (is_add)
1661     {
1662       if (match_local)
1663         return VNET_API_ERROR_VALUE_EXIST;
1664
1665       pool_get (m->locals, local);
1666       clib_memset (local, 0, sizeof (*local));
1667       local->addr.as_u32 = l_addr.as_u32;
1668       local->port = l_port;
1669       local->probability = probability;
1670       local->vrf_id = vrf_id;
1671       local->fib_index =
1672         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1673                                            nat_fib_src_low);
1674
1675       if (!is_out2in_only_static_mapping (m))
1676         {
1677           m_key.addr = l_addr;
1678           m_key.port = l_port;
1679           m_key.fib_index = local->fib_index;
1680           kv.key = m_key.as_u64;
1681           kv.value = m - sm->static_mappings;
1682           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1683             nat_elog_err ("static_mapping_by_local key add failed");
1684         }
1685     }
1686   else
1687     {
1688       if (!match_local)
1689         return VNET_API_ERROR_NO_SUCH_ENTRY;
1690
1691       if (pool_elts (m->locals) < 3)
1692         return VNET_API_ERROR_UNSPECIFIED;
1693
1694       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1695                         nat_fib_src_low);
1696
1697       if (!is_out2in_only_static_mapping (m))
1698         {
1699           m_key.addr = l_addr;
1700           m_key.port = l_port;
1701           m_key.fib_index = match_local->fib_index;
1702           kv.key = m_key.as_u64;
1703           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1704             nat_elog_err ("static_mapping_by_local key del failed");
1705         }
1706
1707       if (sm->num_workers > 1)
1708         {
1709           ip4_header_t ip = {
1710             .src_address = local->addr,
1711           };
1712           tsm = vec_elt_at_index (sm->per_thread_data,
1713                                   sm->worker_in2out_cb (&ip, m->fib_index,
1714                                                         0));
1715         }
1716       else
1717         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1718
1719       /* Delete sessions */
1720       /* *INDENT-OFF* */
1721       pool_foreach (s, tsm->sessions, {
1722         if (!(is_lb_session (s)))
1723           continue;
1724
1725         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1726             (clib_net_to_host_u16 (s->in2out.port) != match_local->port))
1727           continue;
1728
1729         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1730         nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
1731       });
1732       /* *INDENT-ON* */
1733
1734       pool_put (m->locals, match_local);
1735     }
1736
1737   vec_free (m->workers);
1738
1739   /* *INDENT-OFF* */
1740   pool_foreach (local, m->locals,
1741   ({
1742     vec_add1 (locals, local - m->locals);
1743     if (sm->num_workers > 1)
1744       {
1745         ip4_header_t ip;
1746         ip.src_address.as_u32 = local->addr.as_u32,
1747         bitmap = clib_bitmap_set (bitmap,
1748                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1749                                   1);
1750       }
1751   }));
1752   /* *INDENT-ON* */
1753
1754   ASSERT (vec_len (locals) > 1);
1755
1756   local = pool_elt_at_index (m->locals, locals[0]);
1757   local->prefix = local->probability;
1758   for (i = 1; i < vec_len (locals); i++)
1759     {
1760       local = pool_elt_at_index (m->locals, locals[i]);
1761       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1762       local->prefix = local->probability + prev_local->prefix;
1763     }
1764
1765   /* Assign workers */
1766   if (sm->num_workers > 1)
1767     {
1768       /* *INDENT-OFF* */
1769       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1770       /* *INDENT-ON* */
1771     }
1772
1773   return 0;
1774 }
1775
1776 int
1777 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1778                   u8 twice_nat)
1779 {
1780   snat_address_t *a = 0;
1781   snat_session_t *ses;
1782   u32 *ses_to_be_removed = 0, *ses_index;
1783   snat_main_per_thread_data_t *tsm;
1784   snat_static_mapping_t *m;
1785   snat_interface_t *interface;
1786   int i;
1787   snat_address_t *addresses =
1788     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1789
1790   /* Find SNAT address */
1791   for (i = 0; i < vec_len (addresses); i++)
1792     {
1793       if (addresses[i].addr.as_u32 == addr.as_u32)
1794         {
1795           a = addresses + i;
1796           break;
1797         }
1798     }
1799   if (!a)
1800     return VNET_API_ERROR_NO_SUCH_ENTRY;
1801
1802   if (delete_sm)
1803     {
1804       /* *INDENT-OFF* */
1805       pool_foreach (m, sm->static_mappings,
1806       ({
1807           if (m->external_addr.as_u32 == addr.as_u32)
1808             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1809                                             m->local_port, m->external_port,
1810                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1811                                             m->proto, 0, m->twice_nat,
1812                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1813       }));
1814       /* *INDENT-ON* */
1815     }
1816   else
1817     {
1818       /* Check if address is used in some static mapping */
1819       if (is_snat_address_used_in_static_mapping (sm, addr))
1820         {
1821           nat_elog_notice ("address used in static mapping");
1822           return VNET_API_ERROR_UNSPECIFIED;
1823         }
1824     }
1825
1826   if (a->fib_index != ~0)
1827     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1828
1829   /* Delete sessions using address */
1830   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1831     {
1832       /* *INDENT-OFF* */
1833       vec_foreach (tsm, sm->per_thread_data)
1834         {
1835           pool_foreach (ses, tsm->sessions, ({
1836             if (ses->out2in.addr.as_u32 == addr.as_u32)
1837               {
1838                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1839                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1840               }
1841           }));
1842
1843           if (sm->endpoint_dependent){
1844               vec_foreach (ses_index, ses_to_be_removed)
1845                 {
1846                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1847                   nat44_ed_delete_session (sm, ses, tsm - sm->per_thread_data, 1);
1848                 }
1849           }else{
1850               vec_foreach (ses_index, ses_to_be_removed)
1851                 {
1852                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1853                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1854                 }
1855           }
1856
1857           vec_free (ses_to_be_removed);
1858         }
1859       /* *INDENT-ON* */
1860     }
1861
1862 #define _(N, i, n, s) \
1863   vec_free (a->busy_##n##_ports_per_thread);
1864   foreach_snat_protocol
1865 #undef _
1866     if (twice_nat)
1867     {
1868       vec_del1 (sm->twice_nat_addresses, i);
1869       return 0;
1870     }
1871   else
1872     vec_del1 (sm->addresses, i);
1873
1874   /* Delete external address from FIB */
1875   /* *INDENT-OFF* */
1876   pool_foreach (interface, sm->interfaces,
1877   ({
1878     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1879       continue;
1880
1881     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1882     break;
1883   }));
1884   pool_foreach (interface, sm->output_feature_interfaces,
1885   ({
1886     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1887       continue;
1888
1889     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1890     break;
1891   }));
1892   /* *INDENT-ON* */
1893
1894   return 0;
1895 }
1896
1897 int
1898 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1899 {
1900   snat_main_t *sm = &snat_main;
1901   snat_interface_t *i;
1902   const char *feature_name, *del_feature_name;
1903   snat_address_t *ap;
1904   snat_static_mapping_t *m;
1905   snat_det_map_t *dm;
1906   nat_outside_fib_t *outside_fib;
1907   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1908                                                        sw_if_index);
1909
1910   if (sm->out2in_dpo && !is_inside)
1911     return VNET_API_ERROR_UNSUPPORTED;
1912
1913   /* *INDENT-OFF* */
1914   pool_foreach (i, sm->output_feature_interfaces,
1915   ({
1916     if (i->sw_if_index == sw_if_index)
1917       return VNET_API_ERROR_VALUE_EXIST;
1918   }));
1919   /* *INDENT-ON* */
1920
1921   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1922     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1923   else
1924     {
1925       if (sm->num_workers > 1 && !sm->deterministic)
1926         feature_name =
1927           is_inside ? "nat44-in2out-worker-handoff" :
1928           "nat44-out2in-worker-handoff";
1929       else if (sm->deterministic)
1930         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1931       else if (sm->endpoint_dependent)
1932         {
1933           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1934         }
1935       else
1936         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1937     }
1938
1939   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1940     sm->fq_in2out_index =
1941       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1942
1943   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1944     sm->fq_out2in_index =
1945       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1946
1947   if (!is_inside)
1948     {
1949       /* *INDENT-OFF* */
1950       vec_foreach (outside_fib, sm->outside_fibs)
1951         {
1952           if (outside_fib->fib_index == fib_index)
1953             {
1954               if (is_del)
1955                 {
1956                   outside_fib->refcount--;
1957                   if (!outside_fib->refcount)
1958                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1959                 }
1960               else
1961                 outside_fib->refcount++;
1962               goto feature_set;
1963             }
1964         }
1965       /* *INDENT-ON* */
1966       if (!is_del)
1967         {
1968           vec_add2 (sm->outside_fibs, outside_fib, 1);
1969           outside_fib->refcount = 1;
1970           outside_fib->fib_index = fib_index;
1971         }
1972     }
1973 feature_set:
1974   /* *INDENT-OFF* */
1975   pool_foreach (i, sm->interfaces,
1976   ({
1977     if (i->sw_if_index == sw_if_index)
1978       {
1979         if (is_del)
1980           {
1981             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1982               {
1983                 if (is_inside)
1984                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1985                 else
1986                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1987
1988                 if (sm->num_workers > 1 && !sm->deterministic)
1989                   {
1990                     del_feature_name = "nat44-handoff-classify";
1991                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1992                                                  "nat44-out2in-worker-handoff";
1993                   }
1994                 else if (sm->deterministic)
1995                   {
1996                     del_feature_name = "nat44-det-classify";
1997                     feature_name = !is_inside ?  "nat44-det-in2out" :
1998                                                  "nat44-det-out2in";
1999                   }
2000                 else if (sm->endpoint_dependent)
2001                   {
2002                     del_feature_name = "nat44-ed-classify";
2003                     feature_name = !is_inside ?  "nat-pre-in2out" :
2004                                                  "nat-pre-out2in";
2005                   }
2006                 else
2007                   {
2008                     del_feature_name = "nat44-classify";
2009                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2010                   }
2011
2012                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2013                 if (rv)
2014                   return rv;
2015                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2016                                              sw_if_index, 0, 0, 0);
2017                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2018                                              sw_if_index, 1, 0, 0);
2019                 if (!is_inside)
2020                   {
2021                     if (sm->endpoint_dependent)
2022                       vnet_feature_enable_disable ("ip4-local",
2023                                                    "nat44-ed-hairpinning",
2024                                                    sw_if_index, 1, 0, 0);
2025                     else if (!sm->deterministic)
2026                       vnet_feature_enable_disable ("ip4-local",
2027                                                    "nat44-hairpinning",
2028                                                    sw_if_index, 1, 0, 0);
2029                   }
2030               }
2031             else
2032               {
2033                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2034                 if (rv)
2035                   return rv;
2036                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2037                                              sw_if_index, 0, 0, 0);
2038                 pool_put (sm->interfaces, i);
2039                 if (is_inside)
2040                   {
2041                     if (sm->endpoint_dependent)
2042                       vnet_feature_enable_disable ("ip4-local",
2043                                                    "nat44-ed-hairpinning",
2044                                                    sw_if_index, 0, 0, 0);
2045                     else if (!sm->deterministic)
2046                       vnet_feature_enable_disable ("ip4-local",
2047                                                    "nat44-hairpinning",
2048                                                    sw_if_index, 0, 0, 0);
2049                   }
2050               }
2051           }
2052         else
2053           {
2054             if ((nat_interface_is_inside(i) && is_inside) ||
2055                 (nat_interface_is_outside(i) && !is_inside))
2056               return 0;
2057
2058             if (sm->num_workers > 1 && !sm->deterministic)
2059               {
2060                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2061                                                  "nat44-out2in-worker-handoff";
2062                 feature_name = "nat44-handoff-classify";
2063               }
2064             else if (sm->deterministic)
2065               {
2066                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
2067                                                  "nat44-det-out2in";
2068                 feature_name = "nat44-det-classify";
2069               }
2070             else if (sm->endpoint_dependent)
2071               {
2072                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2073                                                  "nat-pre-out2in";
2074
2075                 feature_name = "nat44-ed-classify";
2076               }
2077             else
2078               {
2079                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2080                 feature_name = "nat44-classify";
2081               }
2082
2083             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2084             if (rv)
2085               return rv;
2086             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2087                                          sw_if_index, 0, 0, 0);
2088             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2089                                          sw_if_index, 1, 0, 0);
2090             if (!is_inside)
2091               {
2092                 if (sm->endpoint_dependent)
2093                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2094                                                sw_if_index, 0, 0, 0);
2095                 else if (!sm->deterministic)
2096                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2097                                                sw_if_index, 0, 0, 0);
2098               }
2099             goto set_flags;
2100           }
2101
2102         goto fib;
2103       }
2104   }));
2105   /* *INDENT-ON* */
2106
2107   if (is_del)
2108     return VNET_API_ERROR_NO_SUCH_ENTRY;
2109
2110   pool_get (sm->interfaces, i);
2111   i->sw_if_index = sw_if_index;
2112   i->flags = 0;
2113   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2114                                0);
2115
2116   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2117   if (rv)
2118     return rv;
2119
2120   if (is_inside && !sm->out2in_dpo)
2121     {
2122       if (sm->endpoint_dependent)
2123         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2124                                      sw_if_index, 1, 0, 0);
2125       else if (!sm->deterministic)
2126         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2127                                      sw_if_index, 1, 0, 0);
2128     }
2129
2130 set_flags:
2131   if (is_inside)
2132     {
2133       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2134       return 0;
2135     }
2136   else
2137     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2138
2139   /* Add/delete external addresses to FIB */
2140 fib:
2141   /* *INDENT-OFF* */
2142   vec_foreach (ap, sm->addresses)
2143     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2144
2145   pool_foreach (m, sm->static_mappings,
2146   ({
2147     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2148       continue;
2149
2150     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2151   }));
2152
2153   pool_foreach (dm, sm->det_maps,
2154   ({
2155     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2156   }));
2157   /* *INDENT-ON* */
2158
2159   return 0;
2160 }
2161
2162 int
2163 snat_interface_add_del_output_feature (u32 sw_if_index,
2164                                        u8 is_inside, int is_del)
2165 {
2166   snat_main_t *sm = &snat_main;
2167   snat_interface_t *i;
2168   snat_address_t *ap;
2169   snat_static_mapping_t *m;
2170   nat_outside_fib_t *outside_fib;
2171   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2172                                                        sw_if_index);
2173
2174
2175   if (sm->deterministic ||
2176       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2177     return VNET_API_ERROR_UNSUPPORTED;
2178
2179   /* *INDENT-OFF* */
2180   pool_foreach (i, sm->interfaces,
2181   ({
2182     if (i->sw_if_index == sw_if_index)
2183       return VNET_API_ERROR_VALUE_EXIST;
2184   }));
2185   /* *INDENT-ON* */
2186
2187   if (!is_inside)
2188     {
2189       /* *INDENT-OFF* */
2190       vec_foreach (outside_fib, sm->outside_fibs)
2191         {
2192           if (outside_fib->fib_index == fib_index)
2193             {
2194               if (is_del)
2195                 {
2196                   outside_fib->refcount--;
2197                   if (!outside_fib->refcount)
2198                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2199                 }
2200               else
2201                 outside_fib->refcount++;
2202               goto feature_set;
2203             }
2204         }
2205       /* *INDENT-ON* */
2206       if (!is_del)
2207         {
2208           vec_add2 (sm->outside_fibs, outside_fib, 1);
2209           outside_fib->refcount = 1;
2210           outside_fib->fib_index = fib_index;
2211         }
2212     }
2213
2214 feature_set:
2215   if (is_inside)
2216     {
2217       if (sm->endpoint_dependent)
2218         {
2219           int rv =
2220             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2221           if (rv)
2222             return rv;
2223           rv =
2224             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2225                                                             !is_del);
2226           if (rv)
2227             return rv;
2228           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2229                                        sw_if_index, !is_del, 0, 0);
2230           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2231                                        sw_if_index, !is_del, 0, 0);
2232         }
2233       else
2234         {
2235           int rv =
2236             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2237           if (rv)
2238             return rv;
2239           rv =
2240             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2241                                                             !is_del);
2242           if (rv)
2243             return rv;
2244           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2245                                        sw_if_index, !is_del, 0, 0);
2246           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2247                                        sw_if_index, !is_del, 0, 0);
2248         }
2249       goto fq;
2250     }
2251
2252   if (sm->num_workers > 1)
2253     {
2254       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2255       if (rv)
2256         return rv;
2257       rv =
2258         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2259       if (rv)
2260         return rv;
2261       vnet_feature_enable_disable ("ip4-unicast",
2262                                    "nat44-out2in-worker-handoff",
2263                                    sw_if_index, !is_del, 0, 0);
2264       vnet_feature_enable_disable ("ip4-output",
2265                                    "nat44-in2out-output-worker-handoff",
2266                                    sw_if_index, !is_del, 0, 0);
2267     }
2268   else
2269     {
2270       if (sm->endpoint_dependent)
2271         {
2272           int rv =
2273             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2274           if (rv)
2275             return rv;
2276           rv =
2277             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2278                                                             !is_del);
2279           if (rv)
2280             return rv;
2281           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2282                                        sw_if_index, !is_del, 0, 0);
2283           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2284                                        sw_if_index, !is_del, 0, 0);
2285         }
2286       else
2287         {
2288           int rv =
2289             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2290           if (rv)
2291             return rv;
2292           rv =
2293             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2294                                                             !is_del);
2295           if (rv)
2296             return rv;
2297           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2298                                        sw_if_index, !is_del, 0, 0);
2299           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2300                                        sw_if_index, !is_del, 0, 0);
2301         }
2302     }
2303
2304 fq:
2305   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2306     sm->fq_in2out_output_index =
2307       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2308
2309   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2310     sm->fq_out2in_index =
2311       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2312
2313   /* *INDENT-OFF* */
2314   pool_foreach (i, sm->output_feature_interfaces,
2315   ({
2316     if (i->sw_if_index == sw_if_index)
2317       {
2318         if (is_del)
2319           pool_put (sm->output_feature_interfaces, i);
2320         else
2321           return VNET_API_ERROR_VALUE_EXIST;
2322
2323         goto fib;
2324       }
2325   }));
2326   /* *INDENT-ON* */
2327
2328   if (is_del)
2329     return VNET_API_ERROR_NO_SUCH_ENTRY;
2330
2331   pool_get (sm->output_feature_interfaces, i);
2332   i->sw_if_index = sw_if_index;
2333   i->flags = 0;
2334   if (is_inside)
2335     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2336   else
2337     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2338
2339   /* Add/delete external addresses to FIB */
2340 fib:
2341   if (is_inside)
2342     return 0;
2343
2344   /* *INDENT-OFF* */
2345   vec_foreach (ap, sm->addresses)
2346     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2347
2348   pool_foreach (m, sm->static_mappings,
2349   ({
2350     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2351       continue;
2352
2353     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2354   }));
2355   /* *INDENT-ON* */
2356
2357   return 0;
2358 }
2359
2360 int
2361 snat_set_workers (uword * bitmap)
2362 {
2363   snat_main_t *sm = &snat_main;
2364   int i, j = 0;
2365
2366   if (sm->num_workers < 2)
2367     return VNET_API_ERROR_FEATURE_DISABLED;
2368
2369   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2370     return VNET_API_ERROR_INVALID_WORKER;
2371
2372   vec_free (sm->workers);
2373   /* *INDENT-OFF* */
2374   clib_bitmap_foreach (i, bitmap,
2375     ({
2376       vec_add1(sm->workers, i);
2377       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2378       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2379       j++;
2380     }));
2381   /* *INDENT-ON* */
2382
2383   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2384   sm->num_snat_thread = _vec_len (sm->workers);
2385
2386   return 0;
2387 }
2388
2389 static void
2390 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2391                          u32 old_fib_index)
2392 {
2393   snat_main_t *sm = &snat_main;
2394   nat_outside_fib_t *outside_fib;
2395   snat_interface_t *i;
2396   u8 is_add = 1;
2397   u8 match = 0;
2398
2399   if (new_fib_index == old_fib_index)
2400     return;
2401
2402   if (!vec_len (sm->outside_fibs))
2403     return;
2404
2405   /* *INDENT-OFF* */
2406   pool_foreach (i, sm->interfaces,
2407     ({
2408       if (i->sw_if_index == sw_if_index)
2409         {
2410           if (!(nat_interface_is_outside (i)))
2411             return;
2412           match = 1;
2413         }
2414     }));
2415
2416   pool_foreach (i, sm->output_feature_interfaces,
2417     ({
2418       if (i->sw_if_index == sw_if_index)
2419         {
2420           if (!(nat_interface_is_outside (i)))
2421             return;
2422           match = 1;
2423         }
2424     }));
2425   /* *INDENT-ON* */
2426
2427   if (!match)
2428     return;
2429
2430   vec_foreach (outside_fib, sm->outside_fibs)
2431   {
2432     if (outside_fib->fib_index == old_fib_index)
2433       {
2434         outside_fib->refcount--;
2435         if (!outside_fib->refcount)
2436           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2437         break;
2438       }
2439   }
2440
2441   vec_foreach (outside_fib, sm->outside_fibs)
2442   {
2443     if (outside_fib->fib_index == new_fib_index)
2444       {
2445         outside_fib->refcount++;
2446         is_add = 0;
2447         break;
2448       }
2449   }
2450
2451   if (is_add)
2452     {
2453       vec_add2 (sm->outside_fibs, outside_fib, 1);
2454       outside_fib->refcount = 1;
2455       outside_fib->fib_index = new_fib_index;
2456     }
2457 }
2458
2459 static void
2460 snat_ip4_table_bind (ip4_main_t * im,
2461                      uword opaque,
2462                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2463 {
2464   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2465 }
2466
2467 static void
2468 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2469                                        uword opaque,
2470                                        u32 sw_if_index,
2471                                        ip4_address_t * address,
2472                                        u32 address_length,
2473                                        u32 if_address_index, u32 is_delete);
2474
2475 static void
2476 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2477                                  uword opaque,
2478                                  u32 sw_if_index,
2479                                  ip4_address_t * address,
2480                                  u32 address_length,
2481                                  u32 if_address_index, u32 is_delete);
2482
2483 static int
2484 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2485                                  u32 fib_index,
2486                                  u32 thread_index,
2487                                  snat_session_key_t * k,
2488                                  u16 port_per_thread, u32 snat_thread_index);
2489
2490 static clib_error_t *
2491 snat_init (vlib_main_t * vm)
2492 {
2493   snat_main_t *sm = &snat_main;
2494   clib_error_t *error = 0;
2495   ip4_main_t *im = &ip4_main;
2496   ip_lookup_main_t *lm = &im->lookup_main;
2497   uword *p;
2498   vlib_thread_registration_t *tr;
2499   vlib_thread_main_t *tm = vlib_get_thread_main ();
2500   uword *bitmap = 0;
2501   u32 i;
2502   ip4_add_del_interface_address_callback_t cb4;
2503   vlib_node_t *node;
2504
2505   sm->vlib_main = vm;
2506   sm->vnet_main = vnet_get_main ();
2507   sm->ip4_main = im;
2508   sm->ip4_lookup_main = lm;
2509   sm->api_main = vlibapi_get_main ();
2510   sm->first_worker_index = 0;
2511   sm->num_workers = 0;
2512   sm->num_snat_thread = 1;
2513   sm->workers = 0;
2514   sm->port_per_thread = 0xffff - 1024;
2515   sm->fq_in2out_index = ~0;
2516   sm->fq_in2out_output_index = ~0;
2517   sm->fq_out2in_index = ~0;
2518
2519   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2520   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2521   sm->forwarding_enabled = 0;
2522   sm->log_class = vlib_log_register_class ("nat", 0);
2523   sm->log_level = SNAT_LOG_ERROR;
2524   sm->mss_clamping = 0;
2525
2526   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2527   sm->error_node_index = node->index;
2528
2529   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2530   sm->pre_in2out_node_index = node->index;
2531   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2532   sm->pre_out2in_node_index = node->index;
2533
2534   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2535   sm->pre_in2out_node_index = node->index;
2536
2537   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2538   sm->pre_out2in_node_index = node->index;
2539
2540   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2541   sm->in2out_node_index = node->index;
2542   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2543   sm->in2out_output_node_index = node->index;
2544   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2545   sm->in2out_fast_node_index = node->index;
2546   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2547   sm->in2out_slowpath_node_index = node->index;
2548   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2549   sm->in2out_slowpath_output_node_index = node->index;
2550
2551   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2552   sm->ed_in2out_node_index = node->index;
2553   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2554   sm->ed_in2out_slowpath_node_index = node->index;
2555
2556   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2557   sm->out2in_node_index = node->index;
2558   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2559   sm->out2in_fast_node_index = node->index;
2560
2561   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2562   sm->ed_out2in_node_index = node->index;
2563   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2564   sm->ed_out2in_slowpath_node_index = node->index;
2565
2566   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2567   sm->det_in2out_node_index = node->index;
2568   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2569   sm->det_out2in_node_index = node->index;
2570
2571   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2572   sm->hairpinning_node_index = node->index;
2573   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2574   sm->hairpin_dst_node_index = node->index;
2575   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2576   sm->hairpin_src_node_index = node->index;
2577   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2578   sm->ed_hairpinning_node_index = node->index;
2579   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2580   sm->ed_hairpin_dst_node_index = node->index;
2581   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2582   sm->ed_hairpin_src_node_index = node->index;
2583
2584   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2585   if (p)
2586     {
2587       tr = (vlib_thread_registration_t *) p[0];
2588       if (tr)
2589         {
2590           sm->num_workers = tr->count;
2591           sm->first_worker_index = tr->first_index;
2592         }
2593     }
2594
2595   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2596
2597   /* Use all available workers by default */
2598   if (sm->num_workers > 1)
2599     {
2600       for (i = 0; i < sm->num_workers; i++)
2601         bitmap = clib_bitmap_set (bitmap, i, 1);
2602       snat_set_workers (bitmap);
2603       clib_bitmap_free (bitmap);
2604     }
2605   else
2606     {
2607       sm->per_thread_data[0].snat_thread_index = 0;
2608     }
2609
2610   error = snat_api_init (vm, sm);
2611   if (error)
2612     return error;
2613
2614   /* Set up the interface address add/del callback */
2615   cb4.function = snat_ip4_add_del_interface_address_cb;
2616   cb4.function_opaque = 0;
2617
2618   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2619
2620   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2621   cb4.function_opaque = 0;
2622
2623   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2624
2625   nat_dpo_module_init ();
2626
2627   /* Init counters */
2628   sm->total_users.name = "total-users";
2629   sm->total_users.stat_segment_name = "/nat44/total-users";
2630   vlib_validate_simple_counter (&sm->total_users, 0);
2631   vlib_zero_simple_counter (&sm->total_users, 0);
2632   sm->total_sessions.name = "total-sessions";
2633   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2634   vlib_validate_simple_counter (&sm->total_sessions, 0);
2635   vlib_zero_simple_counter (&sm->total_sessions, 0);
2636
2637   /* Init IPFIX logging */
2638   snat_ipfix_logging_init (vm);
2639
2640   /* Init NAT64 */
2641   error = nat64_init (vm);
2642   if (error)
2643     return error;
2644
2645   nat66_init (vm);
2646
2647   ip4_table_bind_callback_t cbt4 = {
2648     .function = snat_ip4_table_bind,
2649   };
2650   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2651
2652   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2653                                         FIB_SOURCE_PRIORITY_HI,
2654                                         FIB_SOURCE_BH_SIMPLE);
2655   nat_fib_src_low = fib_source_allocate ("nat-low",
2656                                          FIB_SOURCE_PRIORITY_LOW,
2657                                          FIB_SOURCE_BH_SIMPLE);
2658
2659   return error;
2660 }
2661
2662 VLIB_INIT_FUNCTION (snat_init);
2663
2664 void
2665 snat_free_outside_address_and_port (snat_address_t * addresses,
2666                                     u32 thread_index, snat_session_key_t * k)
2667 {
2668   snat_address_t *a;
2669   u32 address_index;
2670   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2671
2672   for (address_index = 0; address_index < vec_len (addresses);
2673        address_index++)
2674     {
2675       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2676         break;
2677     }
2678
2679   ASSERT (address_index < vec_len (addresses));
2680
2681   a = addresses + address_index;
2682
2683   switch (k->protocol)
2684     {
2685 #define _(N, i, n, s) \
2686     case SNAT_PROTOCOL_##N: \
2687       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2688       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2689       a->busy_##n##_ports--; \
2690       a->busy_##n##_ports_per_thread[thread_index]--; \
2691       break;
2692       foreach_snat_protocol
2693 #undef _
2694     default:
2695       nat_elog_info ("unknown protocol");
2696       return;
2697     }
2698 }
2699
2700 static int
2701 nat_set_outside_address_and_port (snat_address_t * addresses,
2702                                   u32 thread_index, snat_session_key_t * k)
2703 {
2704   snat_address_t *a = 0;
2705   u32 address_index;
2706   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2707
2708   for (address_index = 0; address_index < vec_len (addresses);
2709        address_index++)
2710     {
2711       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2712         continue;
2713
2714       a = addresses + address_index;
2715       switch (k->protocol)
2716         {
2717 #define _(N, j, n, s) \
2718         case SNAT_PROTOCOL_##N: \
2719           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2720             return VNET_API_ERROR_INSTANCE_IN_USE; \
2721           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2722           a->busy_##n##_ports_per_thread[thread_index]++; \
2723           a->busy_##n##_ports++; \
2724           return 0;
2725           foreach_snat_protocol
2726 #undef _
2727         default:
2728           nat_elog_info ("unknown protocol");
2729           return 1;
2730         }
2731     }
2732
2733   return VNET_API_ERROR_NO_SUCH_ENTRY;
2734 }
2735
2736 int
2737 snat_static_mapping_match (snat_main_t * sm,
2738                            snat_session_key_t match,
2739                            snat_session_key_t * mapping,
2740                            u8 by_external,
2741                            u8 * is_addr_only,
2742                            twice_nat_type_t * twice_nat,
2743                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2744                            u8 * is_identity_nat)
2745 {
2746   clib_bihash_kv_8_8_t kv, value;
2747   snat_static_mapping_t *m;
2748   snat_session_key_t m_key;
2749   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2750   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2751   u8 backend_index;
2752   nat44_lb_addr_port_t *local;
2753
2754   m_key.fib_index = match.fib_index;
2755   if (by_external)
2756     {
2757       mapping_hash = &sm->static_mapping_by_external;
2758       m_key.fib_index = 0;
2759     }
2760
2761   m_key.addr = match.addr;
2762   m_key.port = clib_net_to_host_u16 (match.port);
2763   m_key.protocol = match.protocol;
2764
2765   kv.key = m_key.as_u64;
2766
2767   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2768     {
2769       /* Try address only mapping */
2770       m_key.port = 0;
2771       m_key.protocol = 0;
2772       kv.key = m_key.as_u64;
2773       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2774         return 1;
2775     }
2776
2777   m = pool_elt_at_index (sm->static_mappings, value.value);
2778
2779   if (by_external)
2780     {
2781       if (is_lb_static_mapping (m))
2782         {
2783           if (PREDICT_FALSE (lb != 0))
2784             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2785           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2786                                                           match.addr,
2787                                                           match.protocol,
2788                                                           match.port,
2789                                                           &backend_index))
2790             {
2791               local = pool_elt_at_index (m->locals, backend_index);
2792               mapping->addr = local->addr;
2793               mapping->port = clib_host_to_net_u16 (local->port);
2794               mapping->fib_index = local->fib_index;
2795               goto end;
2796             }
2797           // pick locals matching this worker
2798           if (PREDICT_FALSE (sm->num_workers > 1))
2799             {
2800               u32 thread_index = vlib_get_thread_index ();
2801               /* *INDENT-OFF* */
2802               pool_foreach_index (i, m->locals,
2803               ({
2804                 local = pool_elt_at_index (m->locals, i);
2805
2806                 ip4_header_t ip = {
2807                   .src_address = local->addr,
2808                 };
2809
2810                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2811                     thread_index)
2812                   {
2813                     vec_add1 (tmp, i);
2814                   }
2815               }));
2816               /* *INDENT-ON* */
2817               ASSERT (vec_len (tmp) != 0);
2818             }
2819           else
2820             {
2821               /* *INDENT-OFF* */
2822               pool_foreach_index (i, m->locals,
2823               ({
2824                 vec_add1 (tmp, i);
2825               }));
2826               /* *INDENT-ON* */
2827             }
2828           hi = vec_len (tmp) - 1;
2829           local = pool_elt_at_index (m->locals, tmp[hi]);
2830           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2831           while (lo < hi)
2832             {
2833               mid = ((hi - lo) >> 1) + lo;
2834               local = pool_elt_at_index (m->locals, tmp[mid]);
2835               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2836             }
2837           local = pool_elt_at_index (m->locals, tmp[lo]);
2838           if (!(local->prefix >= rand))
2839             return 1;
2840           mapping->addr = local->addr;
2841           mapping->port = clib_host_to_net_u16 (local->port);
2842           mapping->fib_index = local->fib_index;
2843           if (m->affinity)
2844             {
2845               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2846                                                 match.protocol, match.port,
2847                                                 tmp[lo], m->affinity,
2848                                                 m->affinity_per_service_list_head_index))
2849                 nat_elog_info ("create affinity record failed");
2850             }
2851           vec_free (tmp);
2852         }
2853       else
2854         {
2855           if (PREDICT_FALSE (lb != 0))
2856             *lb = NO_LB_NAT;
2857           mapping->fib_index = m->fib_index;
2858           mapping->addr = m->local_addr;
2859           /* Address only mapping doesn't change port */
2860           mapping->port = is_addr_only_static_mapping (m) ? match.port
2861             : clib_host_to_net_u16 (m->local_port);
2862         }
2863       mapping->protocol = m->proto;
2864     }
2865   else
2866     {
2867       mapping->addr = m->external_addr;
2868       /* Address only mapping doesn't change port */
2869       mapping->port = is_addr_only_static_mapping (m) ? match.port
2870         : clib_host_to_net_u16 (m->external_port);
2871       mapping->fib_index = sm->outside_fib_index;
2872     }
2873
2874 end:
2875   if (PREDICT_FALSE (is_addr_only != 0))
2876     *is_addr_only = is_addr_only_static_mapping (m);
2877
2878   if (PREDICT_FALSE (twice_nat != 0))
2879     *twice_nat = m->twice_nat;
2880
2881   if (PREDICT_FALSE (is_identity_nat != 0))
2882     *is_identity_nat = is_identity_static_mapping (m);
2883
2884   return 0;
2885 }
2886
2887 static_always_inline u16
2888 snat_random_port (u16 min, u16 max)
2889 {
2890   snat_main_t *sm = &snat_main;
2891   return min + random_u32 (&sm->random_seed) /
2892     (random_u32_max () / (max - min + 1) + 1);
2893 }
2894
2895 int
2896 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2897                                      u32 fib_index,
2898                                      u32 thread_index,
2899                                      snat_session_key_t * k,
2900                                      u16 port_per_thread,
2901                                      u32 snat_thread_index)
2902 {
2903   snat_main_t *sm = &snat_main;
2904
2905   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2906                                   port_per_thread, snat_thread_index);
2907 }
2908
2909 static int
2910 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2911                                  u32 fib_index,
2912                                  u32 thread_index,
2913                                  snat_session_key_t * k,
2914                                  u16 port_per_thread, u32 snat_thread_index)
2915 {
2916   int i;
2917   snat_address_t *a, *ga = 0;
2918   u32 portnum;
2919
2920   for (i = 0; i < vec_len (addresses); i++)
2921     {
2922       a = addresses + i;
2923       switch (k->protocol)
2924         {
2925 #define _(N, j, n, s) \
2926         case SNAT_PROTOCOL_##N: \
2927           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2928             { \
2929               if (a->fib_index == fib_index) \
2930                 { \
2931                   while (1) \
2932                     { \
2933                       portnum = (port_per_thread * \
2934                         snat_thread_index) + \
2935                         snat_random_port(1, port_per_thread) + 1024; \
2936                       if (a->busy_##n##_port_refcounts[portnum]) \
2937                         continue; \
2938                       --a->busy_##n##_port_refcounts[portnum]; \
2939                       a->busy_##n##_ports_per_thread[thread_index]++; \
2940                       a->busy_##n##_ports++; \
2941                       k->addr = a->addr; \
2942                       k->port = clib_host_to_net_u16(portnum); \
2943                       return 0; \
2944                     } \
2945                 } \
2946               else if (a->fib_index == ~0) \
2947                 { \
2948                   ga = a; \
2949                 } \
2950             } \
2951           break;
2952           foreach_snat_protocol
2953 #undef _
2954         default:
2955           nat_elog_info ("unknown protocol");
2956           return 1;
2957         }
2958
2959     }
2960
2961   if (ga)
2962     {
2963       a = ga;
2964       switch (k->protocol)
2965         {
2966 #define _(N, j, n, s) \
2967         case SNAT_PROTOCOL_##N: \
2968           while (1) \
2969             { \
2970               portnum = (port_per_thread * \
2971                 snat_thread_index) + \
2972                 snat_random_port(1, port_per_thread) + 1024; \
2973               if (a->busy_##n##_port_refcounts[portnum]) \
2974                 continue; \
2975               ++a->busy_##n##_port_refcounts[portnum]; \
2976               a->busy_##n##_ports_per_thread[thread_index]++; \
2977               a->busy_##n##_ports++; \
2978               k->addr = a->addr; \
2979               k->port = clib_host_to_net_u16(portnum); \
2980               return 0; \
2981             }
2982           break;
2983           foreach_snat_protocol
2984 #undef _
2985         default:
2986           nat_elog_info ("unknown protocol");
2987           return 1;
2988         }
2989     }
2990
2991   /* Totally out of translations to use... */
2992   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2993   return 1;
2994 }
2995
2996 static int
2997 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2998                               u32 fib_index,
2999                               u32 thread_index,
3000                               snat_session_key_t * k,
3001                               u16 port_per_thread, u32 snat_thread_index)
3002 {
3003   snat_main_t *sm = &snat_main;
3004   snat_address_t *a = addresses;
3005   u16 m, ports, portnum, A, j;
3006   m = 16 - (sm->psid_offset + sm->psid_length);
3007   ports = (1 << (16 - sm->psid_length)) - (1 << m);
3008
3009   if (!vec_len (addresses))
3010     goto exhausted;
3011
3012   switch (k->protocol)
3013     {
3014 #define _(N, i, n, s) \
3015     case SNAT_PROTOCOL_##N: \
3016       if (a->busy_##n##_ports < ports) \
3017         { \
3018           while (1) \
3019             { \
3020               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3021               j = snat_random_port(0, pow2_mask(m)); \
3022               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3023               if (a->busy_##n##_port_refcounts[portnum]) \
3024                 continue; \
3025               ++a->busy_##n##_port_refcounts[portnum]; \
3026               a->busy_##n##_ports++; \
3027               k->addr = a->addr; \
3028               k->port = clib_host_to_net_u16 (portnum); \
3029               return 0; \
3030             } \
3031         } \
3032       break;
3033       foreach_snat_protocol
3034 #undef _
3035     default:
3036       nat_elog_info ("unknown protocol");
3037       return 1;
3038     }
3039
3040 exhausted:
3041   /* Totally out of translations to use... */
3042   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3043   return 1;
3044 }
3045
3046 static int
3047 nat_alloc_addr_and_port_range (snat_address_t * addresses,
3048                                u32 fib_index,
3049                                u32 thread_index,
3050                                snat_session_key_t * k,
3051                                u16 port_per_thread, u32 snat_thread_index)
3052 {
3053   snat_main_t *sm = &snat_main;
3054   snat_address_t *a = addresses;
3055   u16 portnum, ports;
3056
3057   ports = sm->end_port - sm->start_port + 1;
3058
3059   if (!vec_len (addresses))
3060     goto exhausted;
3061
3062   switch (k->protocol)
3063     {
3064 #define _(N, i, n, s) \
3065     case SNAT_PROTOCOL_##N: \
3066       if (a->busy_##n##_ports < ports) \
3067         { \
3068           while (1) \
3069             { \
3070               portnum = snat_random_port(sm->start_port, sm->end_port); \
3071               if (a->busy_##n##_port_refcounts[portnum]) \
3072                 continue; \
3073               ++a->busy_##n##_port_refcounts[portnum]; \
3074               a->busy_##n##_ports++; \
3075               k->addr = a->addr; \
3076               k->port = clib_host_to_net_u16 (portnum); \
3077               return 0; \
3078             } \
3079         } \
3080       break;
3081       foreach_snat_protocol
3082 #undef _
3083     default:
3084       nat_elog_info ("unknown protocol");
3085       return 1;
3086     }
3087
3088 exhausted:
3089   /* Totally out of translations to use... */
3090   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3091   return 1;
3092 }
3093
3094 void
3095 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3096 {
3097   dpo_id_t dpo_v4 = DPO_INVALID;
3098   fib_prefix_t pfx = {
3099     .fp_proto = FIB_PROTOCOL_IP4,
3100     .fp_len = 32,
3101     .fp_addr.ip4.as_u32 = addr.as_u32,
3102   };
3103
3104   if (is_add)
3105     {
3106       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3107       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3108                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3109       dpo_reset (&dpo_v4);
3110     }
3111   else
3112     {
3113       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3114     }
3115 }
3116
3117 u8 *
3118 format_session_kvp (u8 * s, va_list * args)
3119 {
3120   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3121   snat_session_key_t k;
3122
3123   k.as_u64 = v->key;
3124
3125   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
3126
3127   return s;
3128 }
3129
3130 u8 *
3131 format_static_mapping_kvp (u8 * s, va_list * args)
3132 {
3133   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3134   snat_session_key_t k;
3135
3136   k.as_u64 = v->key;
3137
3138   s = format (s, "%U static-mapping-index %llu",
3139               format_static_mapping_key, &k, v->value);
3140
3141   return s;
3142 }
3143
3144 u8 *
3145 format_user_kvp (u8 * s, va_list * args)
3146 {
3147   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3148   snat_user_key_t k;
3149
3150   k.as_u64 = v->key;
3151
3152   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3153               k.fib_index, v->value);
3154
3155   return s;
3156 }
3157
3158 u8 *
3159 format_ed_session_kvp (u8 * s, va_list * args)
3160 {
3161   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3162   nat_ed_ses_key_t k;
3163
3164   k.as_u64[0] = v->key[0];
3165   k.as_u64[1] = v->key[1];
3166
3167   s =
3168     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3169             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
3170             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
3171             format_ip_protocol, k.proto, k.fib_index, v->value);
3172
3173   return s;
3174 }
3175
3176 static u32
3177 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3178                            u8 is_output)
3179 {
3180   snat_main_t *sm = &snat_main;
3181   u32 next_worker_index = 0;
3182   u32 hash;
3183
3184   next_worker_index = sm->first_worker_index;
3185   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3186     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3187
3188   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3189     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3190   else
3191     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3192
3193   return next_worker_index;
3194 }
3195
3196 static u32
3197 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3198                            u32 rx_fib_index0, u8 is_output)
3199 {
3200   snat_main_t *sm = &snat_main;
3201   udp_header_t *udp;
3202   u16 port;
3203   snat_session_key_t m_key;
3204   clib_bihash_kv_8_8_t kv, value;
3205   snat_static_mapping_t *m;
3206   u32 proto;
3207   u32 next_worker_index = 0;
3208
3209   /* first try static mappings without port */
3210   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3211     {
3212       m_key.addr = ip0->dst_address;
3213       m_key.port = 0;
3214       m_key.protocol = 0;
3215       m_key.fib_index = rx_fib_index0;
3216       kv.key = m_key.as_u64;
3217       if (!clib_bihash_search_8_8
3218           (&sm->static_mapping_by_external, &kv, &value))
3219         {
3220           m = pool_elt_at_index (sm->static_mappings, value.value);
3221           return m->workers[0];
3222         }
3223     }
3224
3225   proto = ip_proto_to_snat_proto (ip0->protocol);
3226   udp = ip4_next_header (ip0);
3227   port = udp->dst_port;
3228
3229   /* unknown protocol */
3230   if (PREDICT_FALSE (proto == ~0))
3231     {
3232       /* use current thread */
3233       return vlib_get_thread_index ();
3234     }
3235
3236   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3237     {
3238       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3239       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3240       if (!icmp_type_is_error_message
3241           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3242         port = vnet_buffer (b)->ip.reass.l4_src_port;
3243       else
3244         {
3245           /* if error message, then it's not fragmented and we can access it */
3246           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3247           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3248           void *l4_header = ip4_next_header (inner_ip);
3249           switch (proto)
3250             {
3251             case SNAT_PROTOCOL_ICMP:
3252               icmp = (icmp46_header_t *) l4_header;
3253               echo = (icmp_echo_header_t *) (icmp + 1);
3254               port = echo->identifier;
3255               break;
3256             case SNAT_PROTOCOL_UDP:
3257             case SNAT_PROTOCOL_TCP:
3258               port = ((tcp_udp_header_t *) l4_header)->src_port;
3259               break;
3260             default:
3261               return vlib_get_thread_index ();
3262             }
3263         }
3264     }
3265
3266   /* try static mappings with port */
3267   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3268     {
3269       m_key.addr = ip0->dst_address;
3270       m_key.port = clib_net_to_host_u16 (port);
3271       m_key.protocol = proto;
3272       m_key.fib_index = rx_fib_index0;
3273       kv.key = m_key.as_u64;
3274       if (!clib_bihash_search_8_8
3275           (&sm->static_mapping_by_external, &kv, &value))
3276         {
3277           m = pool_elt_at_index (sm->static_mappings, value.value);
3278           return m->workers[0];
3279         }
3280     }
3281
3282   /* worker by outside port */
3283   next_worker_index = sm->first_worker_index;
3284   next_worker_index +=
3285     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3286   return next_worker_index;
3287 }
3288
3289 static u32
3290 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3291                                u8 is_output)
3292 {
3293   snat_main_t *sm = &snat_main;
3294   u32 next_worker_index = sm->first_worker_index;
3295   u32 hash;
3296
3297   clib_bihash_kv_16_8_t kv16, value16;
3298   snat_main_per_thread_data_t *tsm;
3299   udp_header_t *udp;
3300
3301   if (PREDICT_FALSE (is_output))
3302     {
3303       u32 fib_index = sm->outside_fib_index;
3304       nat_outside_fib_t *outside_fib;
3305       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3306       fib_prefix_t pfx = {
3307         .fp_proto = FIB_PROTOCOL_IP4,
3308         .fp_len = 32,
3309         .fp_addr = {
3310                     .ip4.as_u32 = ip->dst_address.as_u32,
3311                     }
3312         ,
3313       };
3314
3315       udp = ip4_next_header (ip);
3316
3317       switch (vec_len (sm->outside_fibs))
3318         {
3319         case 0:
3320           fib_index = sm->outside_fib_index;
3321           break;
3322         case 1:
3323           fib_index = sm->outside_fibs[0].fib_index;
3324           break;
3325         default:
3326             /* *INDENT-OFF* */
3327             vec_foreach (outside_fib, sm->outside_fibs)
3328               {
3329                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3330                 if (FIB_NODE_INDEX_INVALID != fei)
3331                   {
3332                     if (fib_entry_get_resolving_interface (fei) != ~0)
3333                       {
3334                         fib_index = outside_fib->fib_index;
3335                         break;
3336                       }
3337                   }
3338               }
3339             /* *INDENT-ON* */
3340           break;
3341         }
3342
3343       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3344                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3345
3346       /* *INDENT-OFF* */
3347       vec_foreach (tsm, sm->per_thread_data)
3348         {
3349           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3350                                                       &kv16, &value16)))
3351             {
3352               next_worker_index += tsm->thread_index;
3353
3354               nat_elog_debug_handoff (
3355                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3356                 next_worker_index, fib_index,
3357                 clib_net_to_host_u32 (ip->src_address.as_u32),
3358                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3359
3360               return next_worker_index;
3361             }
3362         }
3363       /* *INDENT-ON* */
3364     }
3365
3366   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3367     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3368
3369   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3370     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3371   else
3372     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3373
3374   if (PREDICT_TRUE (!is_output))
3375     {
3376       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3377                               next_worker_index, rx_fib_index,
3378                               clib_net_to_host_u32 (ip->src_address.as_u32),
3379                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3380     }
3381   else
3382     {
3383       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3384                               next_worker_index, rx_fib_index,
3385                               clib_net_to_host_u32 (ip->src_address.as_u32),
3386                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3387     }
3388
3389   return next_worker_index;
3390 }
3391
3392 static u32
3393 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3394                                u32 rx_fib_index, u8 is_output)
3395 {
3396   snat_main_t *sm = &snat_main;
3397   clib_bihash_kv_8_8_t kv, value;
3398   clib_bihash_kv_16_8_t kv16, value16;
3399   snat_main_per_thread_data_t *tsm;
3400
3401   u32 proto, next_worker_index = 0;
3402   udp_header_t *udp;
3403   u16 port;
3404   snat_static_mapping_t *m;
3405   u32 hash;
3406
3407   proto = ip_proto_to_snat_proto (ip->protocol);
3408
3409   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3410     {
3411       udp = ip4_next_header (ip);
3412
3413       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3414                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3415
3416       /* *INDENT-OFF* */
3417       vec_foreach (tsm, sm->per_thread_data)
3418         {
3419           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3420                                                       &kv16, &value16)))
3421             {
3422               next_worker_index = sm->first_worker_index + tsm->thread_index;
3423               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3424                           next_worker_index, rx_fib_index,
3425                           clib_net_to_host_u32 (ip->src_address.as_u32),
3426                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3427               return next_worker_index;
3428             }
3429           }
3430         /* *INDENT-ON* */
3431     }
3432   else if (proto == SNAT_PROTOCOL_ICMP)
3433     {
3434       nat_ed_ses_key_t key;
3435
3436       if (!get_icmp_o2i_ed_key (b, ip, &key))
3437         {
3438
3439           key.fib_index = rx_fib_index;
3440           kv16.key[0] = key.as_u64[0];
3441           kv16.key[1] = key.as_u64[1];
3442
3443           /* *INDENT-OFF* */
3444           vec_foreach (tsm, sm->per_thread_data)
3445             {
3446               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3447                                                           &kv16, &value16)))
3448                 {
3449                   next_worker_index = sm->first_worker_index +
3450                                       tsm->thread_index;
3451                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3452                               next_worker_index, rx_fib_index,
3453                               clib_net_to_host_u32 (ip->src_address.as_u32),
3454                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3455                   return next_worker_index;
3456                 }
3457             }
3458           /* *INDENT-ON* */
3459         }
3460     }
3461
3462   /* first try static mappings without port */
3463   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3464     {
3465       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3466       if (!clib_bihash_search_8_8
3467           (&sm->static_mapping_by_external, &kv, &value))
3468         {
3469           m = pool_elt_at_index (sm->static_mappings, value.value);
3470           next_worker_index = m->workers[0];
3471           goto done;
3472         }
3473     }
3474
3475   /* unknown protocol */
3476   if (PREDICT_FALSE (proto == ~0))
3477     {
3478       /* use current thread */
3479       next_worker_index = vlib_get_thread_index ();
3480       goto done;
3481     }
3482
3483   udp = ip4_next_header (ip);
3484   port = udp->dst_port;
3485
3486   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3487     {
3488       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3489       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3490       if (!icmp_type_is_error_message
3491           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3492         port = vnet_buffer (b)->ip.reass.l4_src_port;
3493       else
3494         {
3495           /* if error message, then it's not fragmented and we can access it */
3496           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3497           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3498           void *l4_header = ip4_next_header (inner_ip);
3499           switch (proto)
3500             {
3501             case SNAT_PROTOCOL_ICMP:
3502               icmp = (icmp46_header_t *) l4_header;
3503               echo = (icmp_echo_header_t *) (icmp + 1);
3504               port = echo->identifier;
3505               break;
3506             case SNAT_PROTOCOL_UDP:
3507             case SNAT_PROTOCOL_TCP:
3508               port = ((tcp_udp_header_t *) l4_header)->src_port;
3509               break;
3510             default:
3511               next_worker_index = vlib_get_thread_index ();
3512               goto done;
3513             }
3514         }
3515     }
3516
3517   /* try static mappings with port */
3518   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3519     {
3520       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3521                   clib_net_to_host_u16 (port));
3522       if (!clib_bihash_search_8_8
3523           (&sm->static_mapping_by_external, &kv, &value))
3524         {
3525           m = pool_elt_at_index (sm->static_mappings, value.value);
3526           if (!is_lb_static_mapping (m))
3527             {
3528               next_worker_index = m->workers[0];
3529               goto done;
3530             }
3531
3532           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3533             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3534
3535           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3536             next_worker_index =
3537               m->workers[hash & (_vec_len (m->workers) - 1)];
3538           else
3539             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3540           goto done;
3541         }
3542     }
3543
3544   /* worker by outside port */
3545   next_worker_index = sm->first_worker_index;
3546   next_worker_index +=
3547     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3548
3549 done:
3550   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3551                           clib_net_to_host_u32 (ip->src_address.as_u32),
3552                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3553   return next_worker_index;
3554 }
3555
3556 void
3557 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3558                 ip4_address_t * out_addr, u16 out_port,
3559                 ip4_address_t * eh_addr, u16 eh_port,
3560                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3561                 u32 fib_index, u16 flags, u32 thread_index)
3562 {
3563   snat_main_t *sm = &snat_main;
3564   snat_session_key_t key;
3565   snat_user_t *u;
3566   snat_session_t *s;
3567   clib_bihash_kv_8_8_t kv;
3568   f64 now = vlib_time_now (sm->vlib_main);
3569   nat_outside_fib_t *outside_fib;
3570   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3571   snat_main_per_thread_data_t *tsm;
3572   fib_prefix_t pfx = {
3573     .fp_proto = FIB_PROTOCOL_IP4,
3574     .fp_len = 32,
3575     .fp_addr = {
3576                 .ip4.as_u32 = eh_addr->as_u32,
3577                 },
3578   };
3579
3580   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3581
3582   key.addr.as_u32 = out_addr->as_u32;
3583   key.port = out_port;
3584   key.protocol = proto;
3585
3586   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3587     {
3588       if (nat_set_outside_address_and_port
3589           (sm->addresses, thread_index, &key))
3590         return;
3591     }
3592
3593   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3594   if (!u)
3595     return;
3596
3597   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3598   if (!s)
3599     return;
3600
3601   s->last_heard = now;
3602   s->flags = flags;
3603   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3604   s->ext_host_port = eh_port;
3605   user_session_increment (sm, u, snat_is_session_static (s));
3606   switch (vec_len (sm->outside_fibs))
3607     {
3608     case 0:
3609       key.fib_index = sm->outside_fib_index;
3610       break;
3611     case 1:
3612       key.fib_index = sm->outside_fibs[0].fib_index;
3613       break;
3614     default:
3615       /* *INDENT-OFF* */
3616       vec_foreach (outside_fib, sm->outside_fibs)
3617         {
3618           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3619           if (FIB_NODE_INDEX_INVALID != fei)
3620             {
3621               if (fib_entry_get_resolving_interface (fei) != ~0)
3622                 {
3623                   key.fib_index = outside_fib->fib_index;
3624                   break;
3625                 }
3626             }
3627         }
3628       /* *INDENT-ON* */
3629       break;
3630     }
3631   s->out2in = key;
3632   kv.key = key.as_u64;
3633   kv.value = s - tsm->sessions;
3634   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3635     nat_elog_warn ("out2in key add failed");
3636
3637   key.addr.as_u32 = in_addr->as_u32;
3638   key.port = in_port;
3639   key.fib_index = fib_index;
3640   s->in2out = key;
3641   kv.key = key.as_u64;
3642   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3643     nat_elog_warn ("in2out key add failed");
3644 }
3645
3646 void
3647 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3648                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3649                 u32 ti)
3650 {
3651   snat_main_t *sm = &snat_main;
3652   snat_session_key_t key;
3653   clib_bihash_kv_8_8_t kv, value;
3654   u32 thread_index;
3655   snat_session_t *s;
3656   snat_main_per_thread_data_t *tsm;
3657
3658   if (sm->num_workers > 1)
3659     thread_index =
3660       sm->first_worker_index +
3661       (sm->workers[(clib_net_to_host_u16 (out_port) -
3662                     1024) / sm->port_per_thread]);
3663   else
3664     thread_index = sm->num_workers;
3665   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3666
3667   key.addr.as_u32 = out_addr->as_u32;
3668   key.port = out_port;
3669   key.protocol = proto;
3670   key.fib_index = fib_index;
3671   kv.key = key.as_u64;
3672   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3673     return;
3674
3675   s = pool_elt_at_index (tsm->sessions, value.value);
3676   nat_free_session_data (sm, s, thread_index, 1);
3677   nat44_delete_session (sm, s, thread_index);
3678 }
3679
3680 void
3681 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3682                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3683                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3684 {
3685   snat_main_t *sm = &snat_main;
3686   snat_session_key_t key;
3687   clib_bihash_kv_8_8_t kv, value;
3688   snat_session_t *s;
3689   snat_main_per_thread_data_t *tsm;
3690
3691   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3692
3693   key.addr.as_u32 = out_addr->as_u32;
3694   key.port = out_port;
3695   key.protocol = proto;
3696   key.fib_index = fib_index;
3697   kv.key = key.as_u64;
3698   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3699     return;
3700
3701   s = pool_elt_at_index (tsm->sessions, value.value);
3702   s->total_pkts = total_pkts;
3703   s->total_bytes = total_bytes;
3704 }
3705
3706 void
3707 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3708                    ip4_address_t * out_addr, u16 out_port,
3709                    ip4_address_t * eh_addr, u16 eh_port,
3710                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3711                    u32 fib_index, u16 flags, u32 thread_index)
3712 {
3713   snat_main_t *sm = &snat_main;
3714   snat_session_key_t key;
3715   snat_session_t *s;
3716   clib_bihash_kv_16_8_t kv;
3717   f64 now = vlib_time_now (sm->vlib_main);
3718   nat_outside_fib_t *outside_fib;
3719   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3720   snat_main_per_thread_data_t *tsm;
3721   fib_prefix_t pfx = {
3722     .fp_proto = FIB_PROTOCOL_IP4,
3723     .fp_len = 32,
3724     .fp_addr = {
3725                 .ip4.as_u32 = eh_addr->as_u32,
3726                 },
3727   };
3728
3729   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3730
3731   key.addr.as_u32 = out_addr->as_u32;
3732   key.port = out_port;
3733   key.protocol = proto;
3734
3735   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3736     {
3737       if (nat_set_outside_address_and_port
3738           (sm->addresses, thread_index, &key))
3739         return;
3740     }
3741
3742   key.addr.as_u32 = ehn_addr->as_u32;
3743   key.port = ehn_port;
3744   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3745     {
3746       if (nat_set_outside_address_and_port
3747           (sm->twice_nat_addresses, thread_index, &key))
3748         return;
3749     }
3750
3751   s = nat_ed_session_alloc (sm, thread_index, now);
3752   if (!s)
3753     return;
3754
3755   s->last_heard = now;
3756   s->flags = flags;
3757   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3758   s->ext_host_nat_port = s->ext_host_port = eh_port;
3759   if (is_twice_nat_session (s))
3760     {
3761       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3762       s->ext_host_nat_port = ehn_port;
3763     }
3764   switch (vec_len (sm->outside_fibs))
3765     {
3766     case 0:
3767       key.fib_index = sm->outside_fib_index;
3768       break;
3769     case 1:
3770       key.fib_index = sm->outside_fibs[0].fib_index;
3771       break;
3772     default:
3773       /* *INDENT-OFF* */
3774       vec_foreach (outside_fib, sm->outside_fibs)
3775         {
3776           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3777           if (FIB_NODE_INDEX_INVALID != fei)
3778             {
3779               if (fib_entry_get_resolving_interface (fei) != ~0)
3780                 {
3781                   key.fib_index = outside_fib->fib_index;
3782                   break;
3783                 }
3784             }
3785         }
3786       /* *INDENT-ON* */
3787       break;
3788     }
3789   key.addr.as_u32 = out_addr->as_u32;
3790   key.port = out_port;
3791   s->out2in = key;
3792   kv.value = s - tsm->sessions;
3793
3794   key.addr.as_u32 = in_addr->as_u32;
3795   key.port = in_port;
3796   key.fib_index = fib_index;
3797   s->in2out = key;
3798
3799   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3800               snat_proto_to_ip_proto (proto), fib_index, in_port,
3801               s->ext_host_nat_port);
3802   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3803     nat_elog_warn ("in2out key add failed");
3804
3805   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3806               s->out2in.fib_index, out_port, eh_port);
3807   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3808     nat_elog_warn ("out2in key add failed");
3809 }
3810
3811 void
3812 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3813                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3814                    u32 fib_index, u32 ti)
3815 {
3816   snat_main_t *sm = &snat_main;
3817   nat_ed_ses_key_t key;
3818   clib_bihash_kv_16_8_t kv, value;
3819   u32 thread_index;
3820   snat_session_t *s;
3821   snat_main_per_thread_data_t *tsm;
3822
3823   if (sm->num_workers > 1)
3824     thread_index =
3825       sm->first_worker_index +
3826       (sm->workers[(clib_net_to_host_u16 (out_port) -
3827                     1024) / sm->port_per_thread]);
3828   else
3829     thread_index = sm->num_workers;
3830   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3831
3832   key.l_addr.as_u32 = out_addr->as_u32;
3833   key.l_port = out_port;
3834   key.r_addr.as_u32 = eh_addr->as_u32;
3835   key.r_port = eh_port;
3836   key.proto = proto;
3837   key.fib_index = fib_index;
3838   kv.key[0] = key.as_u64[0];
3839   kv.key[1] = key.as_u64[1];
3840   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3841     return;
3842
3843   s = pool_elt_at_index (tsm->sessions, value.value);
3844   nat_free_session_data (sm, s, thread_index, 1);
3845   nat44_delete_session (sm, s, thread_index);
3846 }
3847
3848 void
3849 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3850                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3851                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3852                    u32 thread_index)
3853 {
3854   snat_main_t *sm = &snat_main;
3855   nat_ed_ses_key_t key;
3856   clib_bihash_kv_16_8_t kv, value;
3857   snat_session_t *s;
3858   snat_main_per_thread_data_t *tsm;
3859
3860   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3861
3862   key.l_addr.as_u32 = out_addr->as_u32;
3863   key.l_port = out_port;
3864   key.r_addr.as_u32 = eh_addr->as_u32;
3865   key.r_port = eh_port;
3866   key.proto = proto;
3867   key.fib_index = fib_index;
3868   kv.key[0] = key.as_u64[0];
3869   kv.key[1] = key.as_u64[1];
3870   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3871     return;
3872
3873   s = pool_elt_at_index (tsm->sessions, value.value);
3874   s->total_pkts = total_pkts;
3875   s->total_bytes = total_bytes;
3876 }
3877
3878 static clib_error_t *
3879 snat_config (vlib_main_t * vm, unformat_input_t * input)
3880 {
3881   snat_main_t *sm = &snat_main;
3882   nat66_main_t *nm = &nat66_main;
3883   //dslite_main_t *dm = &dslite_main;
3884   snat_main_per_thread_data_t *tsm;
3885
3886   u32 static_mapping_buckets = 1024;
3887   uword static_mapping_memory_size = 64 << 20;
3888
3889   u32 nat64_bib_buckets = 1024;
3890   u32 nat64_bib_memory_size = 128 << 20;
3891
3892   u32 nat64_st_buckets = 2048;
3893   uword nat64_st_memory_size = 256 << 20;
3894
3895   u32 user_buckets = 128;
3896   uword user_memory_size = 64 << 20;
3897   u32 translation_buckets = 1024;
3898   uword translation_memory_size = 128 << 20;
3899
3900   u32 max_translations_per_user = ~0;
3901
3902   u32 outside_vrf_id = 0;
3903   u32 outside_ip6_vrf_id = 0;
3904   u32 inside_vrf_id = 0;
3905   u8 static_mapping_only = 0;
3906   u8 static_mapping_connection_tracking = 0;
3907
3908   // configurable timeouts
3909   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3910   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3911   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3912   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3913
3914   sm->deterministic = 0;
3915   sm->out2in_dpo = 0;
3916   sm->endpoint_dependent = 0;
3917
3918   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3919     {
3920       if (unformat
3921           (input, "translation hash buckets %d", &translation_buckets))
3922         ;
3923       else if (unformat (input, "udp timeout %d", &udp_timeout))
3924         ;
3925       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3926         ;
3927       else if (unformat (input, "tcp transitory timeout %d",
3928                          &tcp_transitory_timeout));
3929       else if (unformat (input, "tcp established timeout %d",
3930                          &tcp_established_timeout));
3931       else if (unformat (input, "translation hash memory %d",
3932                          &translation_memory_size));
3933       else if (unformat (input, "user hash buckets %d", &user_buckets))
3934         ;
3935       else if (unformat (input, "user hash memory %d", &user_memory_size))
3936         ;
3937       else if (unformat (input, "max translations per user %d",
3938                          &max_translations_per_user))
3939         ;
3940       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3941         ;
3942       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3943         ;
3944       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3945         ;
3946       else if (unformat (input, "static mapping only"))
3947         {
3948           static_mapping_only = 1;
3949           if (unformat (input, "connection tracking"))
3950             static_mapping_connection_tracking = 1;
3951         }
3952       else if (unformat (input, "deterministic"))
3953         sm->deterministic = 1;
3954       else if (unformat (input, "nat64 bib hash buckets %d",
3955                          &nat64_bib_buckets))
3956         ;
3957       else if (unformat (input, "nat64 bib hash memory %d",
3958                          &nat64_bib_memory_size))
3959         ;
3960       else
3961         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3962         ;
3963       else if (unformat (input, "nat64 st hash memory %d",
3964                          &nat64_st_memory_size))
3965         ;
3966       else if (unformat (input, "out2in dpo"))
3967         sm->out2in_dpo = 1;
3968       //else if (unformat (input, "dslite ce"))
3969       //dslite_set_ce (dm, 1);
3970       else if (unformat (input, "endpoint-dependent"))
3971         sm->endpoint_dependent = 1;
3972       else
3973         return clib_error_return (0, "unknown input '%U'",
3974                                   format_unformat_error, input);
3975     }
3976
3977   if (sm->deterministic && sm->endpoint_dependent)
3978     return clib_error_return (0,
3979                               "deterministic and endpoint-dependent modes are mutually exclusive");
3980
3981   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3982     return clib_error_return (0,
3983                               "static mapping only mode available only for simple nat");
3984
3985   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3986     return clib_error_return (0,
3987                               "out2in dpo mode available only for simple nat");
3988
3989   /* optionally configurable timeouts for testing purposes */
3990   sm->udp_timeout = udp_timeout;
3991   sm->tcp_transitory_timeout = tcp_transitory_timeout;
3992   sm->tcp_established_timeout = tcp_established_timeout;
3993   sm->icmp_timeout = icmp_timeout;
3994
3995   sm->user_buckets = user_buckets;
3996   sm->user_memory_size = user_memory_size;
3997
3998   sm->translation_buckets = translation_buckets;
3999   sm->translation_memory_size = translation_memory_size;
4000
4001   /* do not exceed load factor 10 */
4002   sm->max_translations = 10 * translation_buckets;
4003   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4004     sm->max_translations : max_translations_per_user;
4005
4006   sm->outside_vrf_id = outside_vrf_id;
4007   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4008                                                              outside_vrf_id,
4009                                                              nat_fib_src_hi);
4010   nm->outside_vrf_id = outside_ip6_vrf_id;
4011   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
4012                                                              outside_ip6_vrf_id,
4013                                                              nat_fib_src_hi);
4014   sm->inside_vrf_id = inside_vrf_id;
4015   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4016                                                             inside_vrf_id,
4017                                                             nat_fib_src_hi);
4018   sm->static_mapping_only = static_mapping_only;
4019   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4020
4021   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4022                   nat64_st_memory_size);
4023
4024   if (sm->deterministic)
4025     {
4026       sm->in2out_node_index = snat_det_in2out_node.index;
4027       sm->in2out_output_node_index = ~0;
4028       sm->out2in_node_index = snat_det_out2in_node.index;
4029       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4030       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4031     }
4032   else
4033     {
4034       if (sm->endpoint_dependent)
4035         {
4036           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4037           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4038
4039           sm->handoff_out2in_index = nat_pre_out2in_node.index;
4040           sm->handoff_in2out_index = nat_pre_in2out_node.index;
4041           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
4042
4043           sm->in2out_node_index = nat44_ed_in2out_node.index;
4044           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4045           sm->out2in_node_index = nat44_ed_out2in_node.index;
4046
4047           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4048           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4049           nat_affinity_init (vm);
4050           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4051                        nat_ha_sref_ed_cb);
4052         }
4053       else
4054         {
4055           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4056           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4057
4058           sm->handoff_out2in_index = snat_out2in_node.index;
4059           sm->handoff_in2out_index = snat_in2out_node.index;
4060           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
4061
4062           sm->in2out_node_index = snat_in2out_node.index;
4063           sm->in2out_output_node_index = snat_in2out_output_node.index;
4064           sm->out2in_node_index = snat_out2in_node.index;
4065           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4066           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4067           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4068         }
4069       if (!static_mapping_only ||
4070           (static_mapping_only && static_mapping_connection_tracking))
4071         {
4072           /* *INDENT-OFF* */
4073           vec_foreach (tsm, sm->per_thread_data)
4074             {
4075               pool_alloc (tsm->sessions, sm->max_translations);
4076               pool_alloc (tsm->list_pool, sm->max_translations);
4077               pool_alloc (tsm->global_lru_pool, sm->max_translations);
4078
4079               dlist_elt_t *head;
4080               pool_get (tsm->global_lru_pool, head);
4081               tsm->global_lru_head_index = head - tsm->global_lru_pool;
4082               clib_dlist_init (tsm->global_lru_pool,
4083                                tsm->global_lru_head_index);
4084
4085               if (sm->endpoint_dependent)
4086                 {
4087                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
4088                                          translation_buckets,
4089                                          translation_memory_size);
4090                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
4091                                                       format_ed_session_kvp);
4092
4093                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
4094                                          translation_buckets,
4095                                          translation_memory_size);
4096                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
4097                                                       format_ed_session_kvp);
4098                 }
4099               else
4100                 {
4101                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
4102                                         translation_buckets,
4103                                         translation_memory_size);
4104                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
4105                                                      format_session_kvp);
4106
4107                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
4108                                         translation_buckets,
4109                                         translation_memory_size);
4110                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
4111                                                      format_session_kvp);
4112                 }
4113
4114               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
4115                                     user_memory_size);
4116               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
4117                                                  format_user_kvp);
4118             }
4119           /* *INDENT-ON* */
4120
4121         }
4122       else
4123         {
4124           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4125           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4126         }
4127       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4128                             "static_mapping_by_local", static_mapping_buckets,
4129                             static_mapping_memory_size);
4130       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4131                                          format_static_mapping_kvp);
4132
4133       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4134                             "static_mapping_by_external",
4135                             static_mapping_buckets,
4136                             static_mapping_memory_size);
4137       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4138                                          format_static_mapping_kvp);
4139     }
4140
4141   return 0;
4142 }
4143
4144 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4145
4146 static void
4147 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4148                                  uword opaque,
4149                                  u32 sw_if_index,
4150                                  ip4_address_t * address,
4151                                  u32 address_length,
4152                                  u32 if_address_index, u32 is_delete)
4153 {
4154   snat_main_t *sm = &snat_main;
4155   snat_static_map_resolve_t *rp;
4156   snat_static_mapping_t *m;
4157   snat_session_key_t m_key;
4158   clib_bihash_kv_8_8_t kv, value;
4159   int i, rv;
4160   ip4_address_t l_addr;
4161
4162   for (i = 0; i < vec_len (sm->to_resolve); i++)
4163     {
4164       rp = sm->to_resolve + i;
4165       if (rp->addr_only == 0)
4166         continue;
4167       if (rp->sw_if_index == sw_if_index)
4168         goto match;
4169     }
4170
4171   return;
4172
4173 match:
4174   m_key.addr.as_u32 = address->as_u32;
4175   m_key.port = rp->addr_only ? 0 : rp->e_port;
4176   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4177   m_key.fib_index = sm->outside_fib_index;
4178   kv.key = m_key.as_u64;
4179   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4180     m = 0;
4181   else
4182     m = pool_elt_at_index (sm->static_mappings, value.value);
4183
4184   if (!is_delete)
4185     {
4186       /* Don't trip over lease renewal, static config */
4187       if (m)
4188         return;
4189     }
4190   else
4191     {
4192       if (!m)
4193         return;
4194     }
4195
4196   /* Indetity mapping? */
4197   if (rp->l_addr.as_u32 == 0)
4198     l_addr.as_u32 = address[0].as_u32;
4199   else
4200     l_addr.as_u32 = rp->l_addr.as_u32;
4201   /* Add the static mapping */
4202   rv = snat_add_static_mapping (l_addr,
4203                                 address[0],
4204                                 rp->l_port,
4205                                 rp->e_port,
4206                                 rp->vrf_id,
4207                                 rp->addr_only, ~0 /* sw_if_index */ ,
4208                                 rp->proto, !is_delete, rp->twice_nat,
4209                                 rp->out2in_only, rp->tag, rp->identity_nat);
4210   if (rv)
4211     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4212 }
4213
4214 static void
4215 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4216                                        uword opaque,
4217                                        u32 sw_if_index,
4218                                        ip4_address_t * address,
4219                                        u32 address_length,
4220                                        u32 if_address_index, u32 is_delete)
4221 {
4222   snat_main_t *sm = &snat_main;
4223   snat_static_map_resolve_t *rp;
4224   ip4_address_t l_addr;
4225   int i, j;
4226   int rv;
4227   u8 twice_nat = 0;
4228   snat_address_t *addresses = sm->addresses;
4229
4230   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4231     {
4232       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4233         goto match;
4234     }
4235
4236   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4237     {
4238       twice_nat = 1;
4239       addresses = sm->twice_nat_addresses;
4240       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4241         goto match;
4242     }
4243
4244   return;
4245
4246 match:
4247   if (!is_delete)
4248     {
4249       /* Don't trip over lease renewal, static config */
4250       for (j = 0; j < vec_len (addresses); j++)
4251         if (addresses[j].addr.as_u32 == address->as_u32)
4252           return;
4253
4254       (void) snat_add_address (sm, address, ~0, twice_nat);
4255       /* Scan static map resolution vector */
4256       for (j = 0; j < vec_len (sm->to_resolve); j++)
4257         {
4258           rp = sm->to_resolve + j;
4259           if (rp->addr_only)
4260             continue;
4261           /* On this interface? */
4262           if (rp->sw_if_index == sw_if_index)
4263             {
4264               /* Indetity mapping? */
4265               if (rp->l_addr.as_u32 == 0)
4266                 l_addr.as_u32 = address[0].as_u32;
4267               else
4268                 l_addr.as_u32 = rp->l_addr.as_u32;
4269               /* Add the static mapping */
4270               rv = snat_add_static_mapping (l_addr,
4271                                             address[0],
4272                                             rp->l_port,
4273                                             rp->e_port,
4274                                             rp->vrf_id,
4275                                             rp->addr_only,
4276                                             ~0 /* sw_if_index */ ,
4277                                             rp->proto,
4278                                             rp->is_add, rp->twice_nat,
4279                                             rp->out2in_only, rp->tag,
4280                                             rp->identity_nat);
4281               if (rv)
4282                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4283                                     "i4", rv);
4284             }
4285         }
4286       return;
4287     }
4288   else
4289     {
4290       (void) snat_del_address (sm, address[0], 1, twice_nat);
4291       return;
4292     }
4293 }
4294
4295
4296 int
4297 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4298                             u8 twice_nat)
4299 {
4300   ip4_main_t *ip4_main = sm->ip4_main;
4301   ip4_address_t *first_int_addr;
4302   snat_static_map_resolve_t *rp;
4303   u32 *indices_to_delete = 0;
4304   int i, j;
4305   u32 *auto_add_sw_if_indices =
4306     twice_nat ? sm->
4307     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4308
4309   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4310     );
4311
4312   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4313     {
4314       if (auto_add_sw_if_indices[i] == sw_if_index)
4315         {
4316           if (is_del)
4317             {
4318               /* if have address remove it */
4319               if (first_int_addr)
4320                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4321               else
4322                 {
4323                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4324                     {
4325                       rp = sm->to_resolve + j;
4326                       if (rp->sw_if_index == sw_if_index)
4327                         vec_add1 (indices_to_delete, j);
4328                     }
4329                   if (vec_len (indices_to_delete))
4330                     {
4331                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4332                         vec_del1 (sm->to_resolve, j);
4333                       vec_free (indices_to_delete);
4334                     }
4335                 }
4336               if (twice_nat)
4337                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4338               else
4339                 vec_del1 (sm->auto_add_sw_if_indices, i);
4340             }
4341           else
4342             return VNET_API_ERROR_VALUE_EXIST;
4343
4344           return 0;
4345         }
4346     }
4347
4348   if (is_del)
4349     return VNET_API_ERROR_NO_SUCH_ENTRY;
4350
4351   /* add to the auto-address list */
4352   if (twice_nat)
4353     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4354   else
4355     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4356
4357   /* If the address is already bound - or static - add it now */
4358   if (first_int_addr)
4359     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4360
4361   return 0;
4362 }
4363
4364 int
4365 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4366                    snat_protocol_t proto, u32 vrf_id, int is_in)
4367 {
4368   snat_main_per_thread_data_t *tsm;
4369   clib_bihash_kv_8_8_t kv, value;
4370   ip4_header_t ip;
4371   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4372   snat_session_key_t key;
4373   snat_session_t *s;
4374   clib_bihash_8_8_t *t;
4375
4376   if (sm->endpoint_dependent)
4377     return VNET_API_ERROR_UNSUPPORTED;
4378
4379   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4380   if (sm->num_workers > 1)
4381     tsm =
4382       vec_elt_at_index (sm->per_thread_data,
4383                         sm->worker_in2out_cb (&ip, fib_index, 0));
4384   else
4385     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4386
4387   key.addr.as_u32 = addr->as_u32;
4388   key.port = clib_host_to_net_u16 (port);
4389   key.protocol = proto;
4390   key.fib_index = fib_index;
4391   kv.key = key.as_u64;
4392   t = is_in ? &tsm->in2out : &tsm->out2in;
4393   if (!clib_bihash_search_8_8 (t, &kv, &value))
4394     {
4395       if (pool_is_free_index (tsm->sessions, value.value))
4396         return VNET_API_ERROR_UNSPECIFIED;
4397
4398       s = pool_elt_at_index (tsm->sessions, value.value);
4399       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4400       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4401       return 0;
4402     }
4403
4404   return VNET_API_ERROR_NO_SUCH_ENTRY;
4405 }
4406
4407 int
4408 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4409                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4410                       u32 vrf_id, int is_in)
4411 {
4412   ip4_header_t ip;
4413   clib_bihash_16_8_t *t;
4414   nat_ed_ses_key_t key;
4415   clib_bihash_kv_16_8_t kv, value;
4416   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4417   snat_session_t *s;
4418   snat_main_per_thread_data_t *tsm;
4419
4420   if (!sm->endpoint_dependent)
4421     return VNET_API_ERROR_FEATURE_DISABLED;
4422
4423   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4424   if (sm->num_workers > 1)
4425     tsm =
4426       vec_elt_at_index (sm->per_thread_data,
4427                         sm->worker_in2out_cb (&ip, fib_index, 0));
4428   else
4429     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4430
4431   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4432   key.l_addr.as_u32 = addr->as_u32;
4433   key.r_addr.as_u32 = eh_addr->as_u32;
4434   key.l_port = clib_host_to_net_u16 (port);
4435   key.r_port = clib_host_to_net_u16 (eh_port);
4436   key.proto = proto;
4437   key.fib_index = fib_index;
4438   kv.key[0] = key.as_u64[0];
4439   kv.key[1] = key.as_u64[1];
4440   if (clib_bihash_search_16_8 (t, &kv, &value))
4441     return VNET_API_ERROR_NO_SUCH_ENTRY;
4442
4443   if (pool_is_free_index (tsm->sessions, value.value))
4444     return VNET_API_ERROR_UNSPECIFIED;
4445   s = pool_elt_at_index (tsm->sessions, value.value);
4446   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4447   nat44_ed_delete_session (sm, s, tsm - sm->per_thread_data, 1);
4448   return 0;
4449 }
4450
4451 void
4452 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4453 {
4454   snat_main_t *sm = &snat_main;
4455
4456   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4457   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4458   sm->psid = psid;
4459   sm->psid_offset = psid_offset;
4460   sm->psid_length = psid_length;
4461 }
4462
4463 void
4464 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4465 {
4466   snat_main_t *sm = &snat_main;
4467
4468   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4469   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4470   sm->start_port = start_port;
4471   sm->end_port = end_port;
4472 }
4473
4474 void
4475 nat_set_alloc_addr_and_port_default (void)
4476 {
4477   snat_main_t *sm = &snat_main;
4478
4479   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4480   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4481 }
4482
4483 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4484                                  vlib_node_runtime_t * node,
4485                                  vlib_frame_t * frame)
4486 {
4487   return 0;
4488 }
4489
4490 /* *INDENT-OFF* */
4491 VLIB_REGISTER_NODE (nat_default_node) = {
4492   .name = "nat-default",
4493   .vector_size = sizeof (u32),
4494   .format_trace = 0,
4495   .type = VLIB_NODE_TYPE_INTERNAL,
4496   .n_errors = 0,
4497   .n_next_nodes = NAT_N_NEXT,
4498   .next_nodes = {
4499     [NAT_NEXT_DROP] = "error-drop",
4500     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4501     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4502     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4503     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4504     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4505     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4506     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4507     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4508   },
4509 };
4510 /* *INDENT-ON* */
4511
4512 /*
4513  * fd.io coding-style-patch-verification: ON
4514  *
4515  * Local Variables:
4516  * eval: (c-set-style "gnu")
4517  * End:
4518  */