src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36 #include <vppinfra/bihash_16_8.h>
  37
  38 #include <vpp/app/version.h>
  39
  40 snat_main_t snat_main;
  41
  42 fib_source_t nat_fib_src_hi;
  43 fib_source_t nat_fib_src_low;
  44
  45 /* *INDENT-OFF* */
  46 /* Hook up input features */
  47 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  48   .arc_name = "ip4-unicast",
  49   .node_name = "nat-pre-in2out",
  50   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  51                                "ip4-sv-reassembly-feature"),
  52 };
  53 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  54   .arc_name = "ip4-unicast",
  55   .node_name = "nat-pre-out2in",
  56   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  57                                "ip4-dhcp-client-detect",
  58                                "ip4-sv-reassembly-feature"),
  59 };
  60 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  61   .arc_name = "ip4-unicast",
  62   .node_name = "nat44-in2out-worker-handoff",
  63   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  64 };
  65 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  66   .arc_name = "ip4-unicast",
  67   .node_name = "nat44-out2in-worker-handoff",
  68   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  69                                "ip4-dhcp-client-detect"),
  70 };
  71 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  72   .arc_name = "ip4-unicast",
  73   .node_name = "nat44-in2out",
  74   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  75 };
  76 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  77   .arc_name = "ip4-unicast",
  78   .node_name = "nat44-out2in",
  79   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  80                                "ip4-dhcp-client-detect"),
  81 };
  82 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  83   .arc_name = "ip4-unicast",
  84   .node_name = "nat44-classify",
  85   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  86 };
  87 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  88   .arc_name = "ip4-unicast",
  89   .node_name = "nat44-det-in2out",
  90   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-det-out2in",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  96                                "ip4-dhcp-client-detect"),
  97 };
  98 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-det-classify",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 102 };
 103 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 104   .arc_name = "ip4-unicast",
 105   .node_name = "nat44-ed-in2out",
 106   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-ed-out2in",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 112                                "ip4-dhcp-client-detect"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-ed-classify",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 118 };
 119 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 120   .arc_name = "ip4-unicast",
 121   .node_name = "nat44-handoff-classify",
 122   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 123 };
 124 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-in2out-fast",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 128 };
 129 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 130   .arc_name = "ip4-unicast",
 131   .node_name = "nat44-out2in-fast",
 132   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 133                                "ip4-dhcp-client-detect"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 136   .arc_name = "ip4-unicast",
 137   .node_name = "nat44-hairpin-dst",
 138   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 141   .arc_name = "ip4-unicast",
 142   .node_name = "nat44-ed-hairpin-dst",
 143   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 144 };
 145
 146 /* Hook up output features */
 147 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 148   .arc_name = "ip4-output",
 149   .node_name = "nat44-in2out-output",
 150   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 151 };
 152 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 153   .arc_name = "ip4-output",
 154   .node_name = "nat44-in2out-output-worker-handoff",
 155   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 156 };
 157 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 158   .arc_name = "ip4-output",
 159   .node_name = "nat44-hairpin-src",
 160   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 161 };
 162 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 163   .arc_name = "ip4-output",
 164   .node_name = "nat44-ed-in2out-output",
 165   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 166   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 167 };
 168 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 169   .arc_name = "ip4-output",
 170   .node_name = "nat44-ed-hairpin-src",
 171   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 172   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 173 };
 174
 175 /* Hook up ip4-local features */
 176 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 177 {
 178   .arc_name = "ip4-local",
 179   .node_name = "nat44-hairpinning",
 180   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 181 };
 182 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 183 {
 184   .arc_name = "ip4-local",
 185   .node_name = "nat44-ed-hairpinning",
 186   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 187 };
 188
 189
 190 VLIB_PLUGIN_REGISTER () = {
 191     .version = VPP_BUILD_VER,
 192     .description = "Network Address Translation (NAT)",
 193 };
 194 /* *INDENT-ON* */
 195
 196 void
 197 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 198                        u8 is_ha)
 199 {
 200   snat_session_key_t key;
 201   clib_bihash_kv_8_8_t kv;
 202   nat_ed_ses_key_t ed_key;
 203   clib_bihash_kv_16_8_t ed_kv;
 204   snat_main_per_thread_data_t *tsm =
 205     vec_elt_at_index (sm->per_thread_data, thread_index);
 206
 207   if (is_fwd_bypass_session (s))
 208     {
 209       if (snat_is_unk_proto_session (s))
 210         {
 211           ed_key.proto = s->in2out.port;
 212           ed_key.r_port = 0;
 213           ed_key.l_port = 0;
 214         }
 215       else
 216         {
 217           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 218           ed_key.l_port = s->in2out.port;
 219           ed_key.r_port = s->ext_host_port;
 220         }
 221       ed_key.l_addr = s->in2out.addr;
 222       ed_key.r_addr = s->ext_host_addr;
 223       ed_key.fib_index = 0;
 224       ed_kv.key[0] = ed_key.as_u64[0];
 225       ed_kv.key[1] = ed_key.as_u64[1];
 226       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 227         nat_elog_warn ("in2out_ed key del failed");
 228       return;
 229     }
 230
 231   /* session lookup tables */
 232   if (is_ed_session (s))
 233     {
 234       if (is_affinity_sessions (s))
 235         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 236                              s->in2out.protocol, s->out2in.port);
 237       ed_key.l_addr = s->out2in.addr;
 238       ed_key.r_addr = s->ext_host_addr;
 239       ed_key.fib_index = s->out2in.fib_index;
 240       if (snat_is_unk_proto_session (s))
 241         {
 242           ed_key.proto = s->in2out.port;
 243           ed_key.r_port = 0;
 244           ed_key.l_port = 0;
 245         }
 246       else
 247         {
 248           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 249           ed_key.l_port = s->out2in.port;
 250           ed_key.r_port = s->ext_host_port;
 251         }
 252       ed_kv.key[0] = ed_key.as_u64[0];
 253       ed_kv.key[1] = ed_key.as_u64[1];
 254       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 255         nat_elog_warn ("out2in_ed key del failed");
 256       ed_key.l_addr = s->in2out.addr;
 257       ed_key.fib_index = s->in2out.fib_index;
 258       if (!snat_is_unk_proto_session (s))
 259         ed_key.l_port = s->in2out.port;
 260       if (is_twice_nat_session (s))
 261         {
 262           ed_key.r_addr = s->ext_host_nat_addr;
 263           ed_key.r_port = s->ext_host_nat_port;
 264         }
 265       ed_kv.key[0] = ed_key.as_u64[0];
 266       ed_kv.key[1] = ed_key.as_u64[1];
 267       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 268         nat_elog_warn ("in2out_ed key del failed");
 269
 270       if (!is_ha)
 271         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 272                                &s->in2out.addr, s->in2out.port,
 273                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 274                                &s->out2in.addr, s->out2in.port,
 275                                &s->ext_host_addr, s->ext_host_port,
 276                                s->in2out.protocol, is_twice_nat_session (s));
 277     }
 278   else
 279     {
 280       kv.key = s->in2out.as_u64;
 281       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 282         nat_elog_warn ("in2out key del failed");
 283       kv.key = s->out2in.as_u64;
 284       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 285         nat_elog_warn ("out2in key del failed");
 286
 287       if (!is_ha)
 288         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 289                                  &s->in2out.addr, s->in2out.port,
 290                                  &s->out2in.addr, s->out2in.port,
 291                                  s->in2out.protocol);
 292     }
 293
 294   if (snat_is_unk_proto_session (s))
 295     return;
 296
 297   if (!is_ha)
 298     {
 299       /* log NAT event */
 300       snat_ipfix_logging_nat44_ses_delete (thread_index,
 301                                            s->in2out.addr.as_u32,
 302                                            s->out2in.addr.as_u32,
 303                                            s->in2out.protocol,
 304                                            s->in2out.port,
 305                                            s->out2in.port,
 306                                            s->in2out.fib_index);
 307
 308       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 309                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 310                    thread_index);
 311     }
 312
 313   /* Twice NAT address and port for external host */
 314   if (is_twice_nat_session (s))
 315     {
 316       key.protocol = s->in2out.protocol;
 317       key.port = s->ext_host_nat_port;
 318       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 319       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 320                                           thread_index, &key);
 321     }
 322
 323   if (snat_is_session_static (s))
 324     return;
 325
 326   ed_bihash_kv_t bihash_key;
 327   clib_memset (&bihash_key, 0, sizeof (bihash_key));
 328   bihash_key.k.dst_address = s->ext_host_addr.as_u32;
 329   bihash_key.k.dst_port = s->ext_host_port;
 330   bihash_key.k.src_address = s->out2in.addr.as_u32;
 331   bihash_key.k.src_port = s->out2in.port;
 332   bihash_key.k.protocol = s->out2in.protocol;
 333   clib_bihash_add_del_16_8 (&sm->ed_ext_ports, &bihash_key.kv,
 334                             0 /* is_add */ );
 335
 336   snat_free_outside_address_and_port (sm->addresses, thread_index,
 337                                       &s->out2in);
 338 }
 339
 340 void
 341 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 342                          u32 thread_index, u8 is_ha)
 343 {
 344   snat_session_key_t key;
 345   nat_ed_ses_key_t ed_key;
 346   clib_bihash_kv_16_8_t ed_kv;
 347   snat_main_per_thread_data_t *tsm =
 348     vec_elt_at_index (sm->per_thread_data, thread_index);
 349
 350   if (is_fwd_bypass_session (s))
 351     {
 352       if (snat_is_unk_proto_session (s))
 353         {
 354           ed_key.proto = s->in2out.port;
 355           ed_key.r_port = 0;
 356           ed_key.l_port = 0;
 357         }
 358       else
 359         {
 360           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 361           ed_key.l_port = s->in2out.port;
 362           ed_key.r_port = s->ext_host_port;
 363         }
 364
 365       ed_key.l_addr = s->in2out.addr;
 366       ed_key.r_addr = s->ext_host_addr;
 367       ed_key.fib_index = 0;
 368       ed_kv.key[0] = ed_key.as_u64[0];
 369       ed_kv.key[1] = ed_key.as_u64[1];
 370
 371       if (PREDICT_FALSE
 372           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 373         nat_elog_warn ("in2out_ed key del failed");
 374       return;
 375     }
 376
 377   /* session lookup tables */
 378   if (is_affinity_sessions (s))
 379     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 380                          s->in2out.protocol, s->out2in.port);
 381   ed_key.l_addr = s->out2in.addr;
 382   ed_key.r_addr = s->ext_host_addr;
 383   ed_key.fib_index = s->out2in.fib_index;
 384   if (snat_is_unk_proto_session (s))
 385     {
 386       ed_key.proto = s->in2out.port;
 387       ed_key.r_port = 0;
 388       ed_key.l_port = 0;
 389     }
 390   else
 391     {
 392       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 393       ed_key.l_port = s->out2in.port;
 394       ed_key.r_port = s->ext_host_port;
 395     }
 396   ed_kv.key[0] = ed_key.as_u64[0];
 397   ed_kv.key[1] = ed_key.as_u64[1];
 398
 399   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0)))
 400     nat_elog_warn ("out2in_ed key del failed");
 401
 402   ed_key.l_addr = s->in2out.addr;
 403   ed_key.fib_index = s->in2out.fib_index;
 404
 405   if (!snat_is_unk_proto_session (s))
 406     ed_key.l_port = s->in2out.port;
 407
 408   if (is_twice_nat_session (s))
 409     {
 410       ed_key.r_addr = s->ext_host_nat_addr;
 411       ed_key.r_port = s->ext_host_nat_port;
 412     }
 413
 414   ed_kv.key[0] = ed_key.as_u64[0];
 415   ed_kv.key[1] = ed_key.as_u64[1];
 416
 417   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 418     nat_elog_warn ("in2out_ed key del failed");
 419
 420   if (!is_ha)
 421     {
 422       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 423                              &s->in2out.addr, s->in2out.port,
 424                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 425                              &s->out2in.addr, s->out2in.port,
 426                              &s->ext_host_addr, s->ext_host_port,
 427                              s->in2out.protocol, is_twice_nat_session (s));
 428     }
 429
 430   if (snat_is_unk_proto_session (s))
 431     return;
 432
 433   // is this correct ?
 434   if (!is_ha)
 435     {
 436       snat_ipfix_logging_nat44_ses_delete (thread_index,
 437                                            s->in2out.addr.as_u32,
 438                                            s->out2in.addr.as_u32,
 439                                            s->in2out.protocol,
 440                                            s->in2out.port,
 441                                            s->out2in.port,
 442                                            s->in2out.fib_index);
 443       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 444                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 445                    thread_index);
 446     }
 447
 448   /* Twice NAT address and port for external host */
 449   if (is_twice_nat_session (s))
 450     {
 451       key.protocol = s->in2out.protocol;
 452       key.port = s->ext_host_nat_port;
 453       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 454       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 455                                           thread_index, &key);
 456     }
 457
 458   if (snat_is_session_static (s))
 459     return;
 460
 461   ed_bihash_kv_t bihash_key;
 462   clib_memset (&bihash_key, 0, sizeof (bihash_key));
 463   bihash_key.k.dst_address = s->ext_host_addr.as_u32;
 464   bihash_key.k.dst_port = s->ext_host_port;
 465   bihash_key.k.src_address = s->out2in.addr.as_u32;
 466   bihash_key.k.src_port = s->out2in.port;
 467   bihash_key.k.protocol = s->out2in.protocol;
 468   clib_bihash_add_del_16_8 (&sm->ed_ext_ports, &bihash_key.kv,
 469                             0 /* is_add */ );
 470
 471   // should be called for every dynamic session
 472   snat_free_outside_address_and_port (sm->addresses, thread_index,
 473                                       &s->out2in);
 474 }
 475
 476
 477 snat_user_t *
 478 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 479                         u32 thread_index)
 480 {
 481   snat_user_t *u = 0;
 482   snat_user_key_t user_key;
 483   clib_bihash_kv_8_8_t kv, value;
 484   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 485   dlist_elt_t *per_user_list_head_elt;
 486
 487   user_key.addr.as_u32 = addr->as_u32;
 488   user_key.fib_index = fib_index;
 489   kv.key = user_key.as_u64;
 490
 491   /* Ever heard of the "user" = src ip4 address before? */
 492   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 493     {
 494       /* no, make a new one */
 495       pool_get (tsm->users, u);
 496       clib_memset (u, 0, sizeof (*u));
 497
 498       u->min_session_timeout = 0;
 499
 500       u->addr.as_u32 = addr->as_u32;
 501       u->fib_index = fib_index;
 502
 503       pool_get (tsm->list_pool, per_user_list_head_elt);
 504
 505       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 506         tsm->list_pool;
 507
 508       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 509
 510       kv.value = u - tsm->users;
 511
 512       /* add user */
 513       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 514         {
 515           nat_elog_warn ("user_hash key add failed");
 516           nat44_delete_user_with_no_session (sm, u, thread_index);
 517           return NULL;
 518         }
 519
 520       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 521                                pool_elts (tsm->users));
 522     }
 523   else
 524     {
 525       u = pool_elt_at_index (tsm->users, value.value);
 526     }
 527
 528   return u;
 529 }
 530
 531 snat_session_t *
 532 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 533                               u32 thread_index, f64 now)
 534 {
 535   snat_session_t *s;
 536   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 537   u32 oldest_per_user_translation_list_index, session_index;
 538   dlist_elt_t *oldest_per_user_translation_list_elt;
 539   dlist_elt_t *per_user_translation_list_elt;
 540
 541   /* Over quota? Recycle the least recently used translation */
 542   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 543     {
 544       oldest_per_user_translation_list_index =
 545         clib_dlist_remove_head (tsm->list_pool,
 546                                 u->sessions_per_user_list_head_index);
 547
 548       ASSERT (oldest_per_user_translation_list_index != ~0);
 549
 550       /* Add it back to the end of the LRU list */
 551       clib_dlist_addtail (tsm->list_pool,
 552                           u->sessions_per_user_list_head_index,
 553                           oldest_per_user_translation_list_index);
 554       /* Get the list element */
 555       oldest_per_user_translation_list_elt =
 556         pool_elt_at_index (tsm->list_pool,
 557                            oldest_per_user_translation_list_index);
 558
 559       /* Get the session index from the list element */
 560       session_index = oldest_per_user_translation_list_elt->value;
 561
 562       /* Get the session */
 563       s = pool_elt_at_index (tsm->sessions, session_index);
 564       nat_free_session_data (sm, s, thread_index, 0);
 565       if (snat_is_session_static (s))
 566         u->nstaticsessions--;
 567       else
 568         u->nsessions--;
 569       s->flags = 0;
 570       s->total_bytes = 0;
 571       s->total_pkts = 0;
 572       s->state = 0;
 573       s->ext_host_addr.as_u32 = 0;
 574       s->ext_host_port = 0;
 575       s->ext_host_nat_addr.as_u32 = 0;
 576       s->ext_host_nat_port = 0;
 577     }
 578   else
 579     {
 580       pool_get (tsm->sessions, s);
 581       clib_memset (s, 0, sizeof (*s));
 582
 583       /* Create list elts */
 584       pool_get (tsm->list_pool, per_user_translation_list_elt);
 585       clib_dlist_init (tsm->list_pool,
 586                        per_user_translation_list_elt - tsm->list_pool);
 587
 588       per_user_translation_list_elt->value = s - tsm->sessions;
 589       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 590       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 591
 592       clib_dlist_addtail (tsm->list_pool,
 593                           s->per_user_list_head_index,
 594                           per_user_translation_list_elt - tsm->list_pool);
 595
 596       s->user_index = u - tsm->users;
 597       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 598                                pool_elts (tsm->sessions));
 599     }
 600
 601   s->ha_last_refreshed = now;
 602
 603   return s;
 604 }
 605
 606 snat_session_t *
 607 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 608                       f64 now)
 609 {
 610   snat_session_t *s;
 611   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 612
 613   dlist_elt_t *oldest_elt;
 614   u64 sess_timeout_time;
 615   u32 oldest_index;
 616
 617   // no sessions
 618   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 619     goto alloc_new;
 620
 621   // no free sessions
 622   if (PREDICT_FALSE
 623       ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user))
 624     {
 625       if (nat44_max_translations_per_user_cleanup (u, thread_index, now))
 626         goto alloc_new;
 627
 628       nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 629                      clib_net_to_host_u32 (u->addr.as_u32));
 630       snat_ipfix_logging_max_entries_per_user (thread_index,
 631                                                sm->max_translations_per_user,
 632                                                u->addr.as_u32);
 633       return 0;
 634     }
 635
 636   oldest_index =
 637     clib_dlist_remove_head (tsm->list_pool,
 638                             u->sessions_per_user_list_head_index);
 639   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 640   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 641
 642   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 643   if (now >= sess_timeout_time ||
 644       (s->tcp_close_timestamp && now >= s->tcp_close_timestamp))
 645     {
 646       // reuse old session
 647       clib_dlist_addtail (tsm->list_pool,
 648                           u->sessions_per_user_list_head_index, oldest_index);
 649       s = nat44_session_reuse_old (sm, u, s, thread_index, now);
 650     }
 651   else
 652     {
 653       // alloc new session
 654       clib_dlist_addhead (tsm->list_pool,
 655                           u->sessions_per_user_list_head_index, oldest_index);
 656     alloc_new:
 657       s = nat44_session_alloc_new (tsm, u, now);
 658       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 659                                pool_elts (tsm->sessions));
 660     }
 661   return s;
 662 }
 663
 664 void
 665 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 666                           int is_add)
 667 {
 668   fib_prefix_t prefix = {
 669     .fp_len = p_len,
 670     .fp_proto = FIB_PROTOCOL_IP4,
 671     .fp_addr = {
 672                 .ip4.as_u32 = addr->as_u32,
 673                 },
 674   };
 675   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 676
 677   if (is_add)
 678     fib_table_entry_update_one_path (fib_index,
 679                                      &prefix,
 680                                      nat_fib_src_low,
 681                                      (FIB_ENTRY_FLAG_CONNECTED |
 682                                       FIB_ENTRY_FLAG_LOCAL |
 683                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 684                                      DPO_PROTO_IP4,
 685                                      NULL,
 686                                      sw_if_index,
 687                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 688   else
 689     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 690 }
 691
 692 int
 693 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 694                   u8 twice_nat)
 695 {
 696   snat_address_t *ap;
 697   snat_interface_t *i;
 698   vlib_thread_main_t *tm = vlib_get_thread_main ();
 699
 700   if (twice_nat && !sm->endpoint_dependent)
 701     return VNET_API_ERROR_FEATURE_DISABLED;
 702
 703   /* Check if address already exists */
 704   /* *INDENT-OFF* */
 705   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 706     {
 707       if (ap->addr.as_u32 == addr->as_u32)
 708         return VNET_API_ERROR_VALUE_EXIST;
 709     }
 710   /* *INDENT-ON* */
 711
 712   if (twice_nat)
 713     vec_add2 (sm->twice_nat_addresses, ap, 1);
 714   else
 715     vec_add2 (sm->addresses, ap, 1);
 716
 717   ap->addr = *addr;
 718   if (vrf_id != ~0)
 719     ap->fib_index =
 720       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 721                                          nat_fib_src_low);
 722   else
 723     ap->fib_index = ~0;
 724 #define _(N, i, n, s) \
 725   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 726   ap->busy_##n##_ports = 0; \
 727   ap->busy_##n##_ports_per_thread = 0;\
 728   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 729   foreach_snat_protocol
 730 #undef _
 731     if (twice_nat)
 732     return 0;
 733
 734   /* Add external address to FIB */
 735   /* *INDENT-OFF* */
 736   pool_foreach (i, sm->interfaces,
 737   ({
 738     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 739       continue;
 740
 741     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 742     break;
 743   }));
 744   pool_foreach (i, sm->output_feature_interfaces,
 745   ({
 746     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 747       continue;
 748
 749     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 750     break;
 751   }));
 752   /* *INDENT-ON* */
 753
 754   return 0;
 755 }
 756
 757 static int
 758 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 759 {
 760   snat_static_mapping_t *m;
 761   /* *INDENT-OFF* */
 762   pool_foreach (m, sm->static_mappings,
 763   ({
 764       if (is_addr_only_static_mapping (m) ||
 765           is_out2in_only_static_mapping (m) ||
 766           is_identity_static_mapping (m))
 767         continue;
 768       if (m->external_addr.as_u32 == addr.as_u32)
 769         return 1;
 770   }));
 771   /* *INDENT-ON* */
 772
 773   return 0;
 774 }
 775
 776 static void
 777 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 778                                        ip4_address_t l_addr,
 779                                        u16 l_port,
 780                                        u32 sw_if_index,
 781                                        u16 e_port,
 782                                        u32 vrf_id,
 783                                        snat_protocol_t proto,
 784                                        int addr_only, int is_add, u8 * tag,
 785                                        int twice_nat, int out2in_only,
 786                                        int identity_nat)
 787 {
 788   snat_static_map_resolve_t *rp;
 789
 790   vec_add2 (sm->to_resolve, rp, 1);
 791   rp->l_addr.as_u32 = l_addr.as_u32;
 792   rp->l_port = l_port;
 793   rp->sw_if_index = sw_if_index;
 794   rp->e_port = e_port;
 795   rp->vrf_id = vrf_id;
 796   rp->proto = proto;
 797   rp->addr_only = addr_only;
 798   rp->is_add = is_add;
 799   rp->twice_nat = twice_nat;
 800   rp->out2in_only = out2in_only;
 801   rp->identity_nat = identity_nat;
 802   rp->tag = vec_dup (tag);
 803 }
 804
 805 static u32
 806 get_thread_idx_by_port (u16 e_port)
 807 {
 808   snat_main_t *sm = &snat_main;
 809   u32 thread_idx = sm->num_workers;
 810   if (sm->num_workers > 1)
 811     {
 812       thread_idx =
 813         sm->first_worker_index +
 814         sm->workers[(e_port - 1024) / sm->port_per_thread];
 815     }
 816   return thread_idx;
 817 }
 818
 819 int
 820 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 821                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 822                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 823                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 824                          u8 identity_nat)
 825 {
 826   snat_main_t *sm = &snat_main;
 827   snat_static_mapping_t *m;
 828   snat_session_key_t m_key;
 829   clib_bihash_kv_8_8_t kv, value;
 830   snat_address_t *a = 0;
 831   u32 fib_index = ~0;
 832   snat_interface_t *interface;
 833   int i;
 834   snat_main_per_thread_data_t *tsm;
 835   snat_user_key_t u_key;
 836   snat_user_t *u;
 837   dlist_elt_t *head, *elt;
 838   u32 elt_index, head_index;
 839   u32 ses_index;
 840   u64 user_index;
 841   snat_session_t *s;
 842   snat_static_map_resolve_t *rp, *rp_match = 0;
 843   nat44_lb_addr_port_t *local;
 844   u32 find = ~0;
 845
 846   if (!sm->endpoint_dependent)
 847     {
 848       if (twice_nat || out2in_only)
 849         return VNET_API_ERROR_FEATURE_DISABLED;
 850     }
 851
 852   /* If the external address is a specific interface address */
 853   if (sw_if_index != ~0)
 854     {
 855       ip4_address_t *first_int_addr;
 856
 857       for (i = 0; i < vec_len (sm->to_resolve); i++)
 858         {
 859           rp = sm->to_resolve + i;
 860           if (rp->sw_if_index != sw_if_index ||
 861               rp->l_addr.as_u32 != l_addr.as_u32 ||
 862               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 863             continue;
 864
 865           if (!addr_only)
 866             {
 867               if ((rp->l_port != l_port && rp->e_port != e_port)
 868                   || rp->proto != proto)
 869                 continue;
 870             }
 871
 872           rp_match = rp;
 873           break;
 874         }
 875
 876       /* Might be already set... */
 877       first_int_addr = ip4_interface_first_address
 878         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 879
 880       if (is_add)
 881         {
 882           if (rp_match)
 883             return VNET_API_ERROR_VALUE_EXIST;
 884
 885           snat_add_static_mapping_when_resolved
 886             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 887              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 888
 889           /* DHCP resolution required? */
 890           if (first_int_addr == 0)
 891             {
 892               return 0;
 893             }
 894           else
 895             {
 896               e_addr.as_u32 = first_int_addr->as_u32;
 897               /* Identity mapping? */
 898               if (l_addr.as_u32 == 0)
 899                 l_addr.as_u32 = e_addr.as_u32;
 900             }
 901         }
 902       else
 903         {
 904           if (!rp_match)
 905             return VNET_API_ERROR_NO_SUCH_ENTRY;
 906
 907           vec_del1 (sm->to_resolve, i);
 908
 909           if (first_int_addr)
 910             {
 911               e_addr.as_u32 = first_int_addr->as_u32;
 912               /* Identity mapping? */
 913               if (l_addr.as_u32 == 0)
 914                 l_addr.as_u32 = e_addr.as_u32;
 915             }
 916           else
 917             return 0;
 918         }
 919     }
 920
 921   m_key.addr = e_addr;
 922   m_key.port = addr_only ? 0 : e_port;
 923   m_key.protocol = addr_only ? 0 : proto;
 924   m_key.fib_index = 0;
 925   kv.key = m_key.as_u64;
 926   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 927     m = 0;
 928   else
 929     m = pool_elt_at_index (sm->static_mappings, value.value);
 930
 931   if (is_add)
 932     {
 933       if (m)
 934         {
 935           if (is_identity_static_mapping (m))
 936             {
 937               /* *INDENT-OFF* */
 938               pool_foreach (local, m->locals,
 939               ({
 940                 if (local->vrf_id == vrf_id)
 941                   return VNET_API_ERROR_VALUE_EXIST;
 942               }));
 943               /* *INDENT-ON* */
 944               pool_get (m->locals, local);
 945               local->vrf_id = vrf_id;
 946               local->fib_index =
 947                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 948                                                    nat_fib_src_low);
 949               m_key.addr = m->local_addr;
 950               m_key.port = m->local_port;
 951               m_key.protocol = m->proto;
 952               m_key.fib_index = local->fib_index;
 953               kv.key = m_key.as_u64;
 954               kv.value = m - sm->static_mappings;
 955               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 956               return 0;
 957             }
 958           else
 959             return VNET_API_ERROR_VALUE_EXIST;
 960         }
 961
 962       if (twice_nat && addr_only)
 963         return VNET_API_ERROR_UNSUPPORTED;
 964
 965       /* Convert VRF id to FIB index */
 966       if (vrf_id != ~0)
 967         fib_index =
 968           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 969                                              nat_fib_src_low);
 970       /* If not specified use inside VRF id from SNAT plugin startup config */
 971       else
 972         {
 973           fib_index = sm->inside_fib_index;
 974           vrf_id = sm->inside_vrf_id;
 975           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 976         }
 977
 978       if (!(out2in_only || identity_nat))
 979         {
 980           m_key.addr = l_addr;
 981           m_key.port = addr_only ? 0 : l_port;
 982           m_key.protocol = addr_only ? 0 : proto;
 983           m_key.fib_index = fib_index;
 984           kv.key = m_key.as_u64;
 985           if (!clib_bihash_search_8_8
 986               (&sm->static_mapping_by_local, &kv, &value))
 987             return VNET_API_ERROR_VALUE_EXIST;
 988         }
 989
 990       /* Find external address in allocated addresses and reserve port for
 991          address and port pair mapping when dynamic translations enabled */
 992       if (!(addr_only || sm->static_mapping_only || out2in_only))
 993         {
 994           for (i = 0; i < vec_len (sm->addresses); i++)
 995             {
 996               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 997                 {
 998                   a = sm->addresses + i;
 999                   /* External port must be unused */
1000                   switch (proto)
1001                     {
1002 #define _(N, j, n, s) \
1003                     case SNAT_PROTOCOL_##N: \
1004                       if (a->busy_##n##_port_refcounts[e_port]) \
1005                         return VNET_API_ERROR_INVALID_VALUE; \
1006                       ++a->busy_##n##_port_refcounts[e_port]; \
1007                       if (e_port > 1024) \
1008                         { \
1009                           a->busy_##n##_ports++; \
1010                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1011                         } \
1012                       break;
1013                       foreach_snat_protocol
1014 #undef _
1015                     default:
1016                       nat_elog_info ("unknown protocol");
1017                       return VNET_API_ERROR_INVALID_VALUE_2;
1018                     }
1019                   break;
1020                 }
1021             }
1022           /* External address must be allocated */
1023           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1024             {
1025               if (sw_if_index != ~0)
1026                 {
1027                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1028                     {
1029                       rp = sm->to_resolve + i;
1030                       if (rp->addr_only)
1031                         continue;
1032                       if (rp->sw_if_index != sw_if_index &&
1033                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1034                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1035                           rp->e_port != e_port && rp->proto != proto)
1036                         continue;
1037
1038                       vec_del1 (sm->to_resolve, i);
1039                       break;
1040                     }
1041                 }
1042               return VNET_API_ERROR_NO_SUCH_ENTRY;
1043             }
1044         }
1045
1046       pool_get (sm->static_mappings, m);
1047       clib_memset (m, 0, sizeof (*m));
1048       m->tag = vec_dup (tag);
1049       m->local_addr = l_addr;
1050       m->external_addr = e_addr;
1051       m->twice_nat = twice_nat;
1052       if (out2in_only)
1053         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1054       if (addr_only)
1055         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1056       if (identity_nat)
1057         {
1058           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1059           pool_get (m->locals, local);
1060           local->vrf_id = vrf_id;
1061           local->fib_index = fib_index;
1062         }
1063       else
1064         {
1065           m->vrf_id = vrf_id;
1066           m->fib_index = fib_index;
1067         }
1068       if (!addr_only)
1069         {
1070           m->local_port = l_port;
1071           m->external_port = e_port;
1072           m->proto = proto;
1073         }
1074
1075       if (sm->num_workers > 1)
1076         {
1077           ip4_header_t ip = {
1078             .src_address = m->local_addr,
1079           };
1080           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1081           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1082         }
1083       else
1084         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1085
1086       m_key.addr = m->local_addr;
1087       m_key.port = m->local_port;
1088       m_key.protocol = m->proto;
1089       m_key.fib_index = fib_index;
1090       kv.key = m_key.as_u64;
1091       kv.value = m - sm->static_mappings;
1092       if (!out2in_only)
1093         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1094
1095       m_key.addr = m->external_addr;
1096       m_key.port = m->external_port;
1097       m_key.fib_index = 0;
1098       kv.key = m_key.as_u64;
1099       kv.value = m - sm->static_mappings;
1100       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1101
1102       /* Delete dynamic sessions matching local address (+ local port) */
1103       if (!(sm->static_mapping_only))
1104         {
1105           u_key.addr = m->local_addr;
1106           u_key.fib_index = m->fib_index;
1107           kv.key = u_key.as_u64;
1108           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1109             {
1110               user_index = value.value;
1111               u = pool_elt_at_index (tsm->users, user_index);
1112               if (u->nsessions)
1113                 {
1114                   head_index = u->sessions_per_user_list_head_index;
1115                   head = pool_elt_at_index (tsm->list_pool, head_index);
1116                   elt_index = head->next;
1117                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1118                   ses_index = elt->value;
1119                   while (ses_index != ~0)
1120                     {
1121                       s = pool_elt_at_index (tsm->sessions, ses_index);
1122                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1123                       ses_index = elt->value;
1124
1125                       if (snat_is_session_static (s))
1126                         continue;
1127
1128                       if (!addr_only
1129                           && (clib_net_to_host_u16 (s->in2out.port) !=
1130                               m->local_port))
1131                         continue;
1132
1133                       nat_free_session_data (sm, s,
1134                                              tsm - sm->per_thread_data, 0);
1135                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1136
1137                       if (!addr_only && !sm->endpoint_dependent)
1138                         break;
1139                     }
1140                 }
1141             }
1142         }
1143     }
1144   else
1145     {
1146       if (!m)
1147         {
1148           if (sw_if_index != ~0)
1149             return 0;
1150           else
1151             return VNET_API_ERROR_NO_SUCH_ENTRY;
1152         }
1153
1154       if (identity_nat)
1155         {
1156           if (vrf_id == ~0)
1157             vrf_id = sm->inside_vrf_id;
1158
1159           /* *INDENT-OFF* */
1160           pool_foreach (local, m->locals,
1161           ({
1162             if (local->vrf_id == vrf_id)
1163               find = local - m->locals;
1164           }));
1165           /* *INDENT-ON* */
1166           if (find == ~0)
1167             return VNET_API_ERROR_NO_SUCH_ENTRY;
1168
1169           local = pool_elt_at_index (m->locals, find);
1170           fib_index = local->fib_index;
1171           pool_put (m->locals, local);
1172         }
1173       else
1174         fib_index = m->fib_index;
1175
1176       /* Free external address port */
1177       if (!(addr_only || sm->static_mapping_only || out2in_only))
1178         {
1179           for (i = 0; i < vec_len (sm->addresses); i++)
1180             {
1181               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1182                 {
1183                   a = sm->addresses + i;
1184                   switch (proto)
1185                     {
1186 #define _(N, j, n, s) \
1187                     case SNAT_PROTOCOL_##N: \
1188                       --a->busy_##n##_port_refcounts[e_port]; \
1189                       if (e_port > 1024) \
1190                         { \
1191                           a->busy_##n##_ports--; \
1192                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1193                         } \
1194                       break;
1195                       foreach_snat_protocol
1196 #undef _
1197                     default:
1198                       nat_elog_info ("unknown protocol");
1199                       return VNET_API_ERROR_INVALID_VALUE_2;
1200                     }
1201                   break;
1202                 }
1203             }
1204         }
1205
1206       if (sm->num_workers > 1)
1207         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1208       else
1209         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1210
1211       m_key.addr = m->local_addr;
1212       m_key.port = m->local_port;
1213       m_key.protocol = m->proto;
1214       m_key.fib_index = fib_index;
1215       kv.key = m_key.as_u64;
1216       if (!out2in_only)
1217         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1218
1219       /* Delete session(s) for static mapping if exist */
1220       if (!(sm->static_mapping_only) ||
1221           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1222         {
1223           u_key.addr = m->local_addr;
1224           u_key.fib_index = fib_index;
1225           kv.key = u_key.as_u64;
1226           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1227             {
1228               user_index = value.value;
1229               u = pool_elt_at_index (tsm->users, user_index);
1230               if (u->nstaticsessions)
1231                 {
1232                   head_index = u->sessions_per_user_list_head_index;
1233                   head = pool_elt_at_index (tsm->list_pool, head_index);
1234                   elt_index = head->next;
1235                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1236                   ses_index = elt->value;
1237                   while (ses_index != ~0)
1238                     {
1239                       s = pool_elt_at_index (tsm->sessions, ses_index);
1240                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1241                       ses_index = elt->value;
1242
1243                       if (!addr_only)
1244                         {
1245                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1246                               (clib_net_to_host_u16 (s->out2in.port) !=
1247                                e_port))
1248                             continue;
1249                         }
1250
1251                       if (is_lb_session (s))
1252                         continue;
1253
1254                       if (!snat_is_session_static (s))
1255                         continue;
1256
1257                       nat_free_session_data (sm, s,
1258                                              tsm - sm->per_thread_data, 0);
1259                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1260
1261                       if (!addr_only && !sm->endpoint_dependent)
1262                         break;
1263                     }
1264                 }
1265             }
1266         }
1267
1268       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1269       if (pool_elts (m->locals))
1270         return 0;
1271
1272       m_key.addr = m->external_addr;
1273       m_key.port = m->external_port;
1274       m_key.fib_index = 0;
1275       kv.key = m_key.as_u64;
1276       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1277
1278       vec_free (m->tag);
1279       vec_free (m->workers);
1280       /* Delete static mapping from pool */
1281       pool_put (sm->static_mappings, m);
1282     }
1283
1284   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1285     return 0;
1286
1287   /* Add/delete external address to FIB */
1288   /* *INDENT-OFF* */
1289   pool_foreach (interface, sm->interfaces,
1290   ({
1291     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1292       continue;
1293
1294     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1295     break;
1296   }));
1297   pool_foreach (interface, sm->output_feature_interfaces,
1298   ({
1299     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1300       continue;
1301
1302     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1303     break;
1304   }));
1305   /* *INDENT-ON* */
1306
1307   return 0;
1308 }
1309
1310 int
1311 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1312                                  snat_protocol_t proto,
1313                                  nat44_lb_addr_port_t * locals, u8 is_add,
1314                                  twice_nat_type_t twice_nat, u8 out2in_only,
1315                                  u8 * tag, u32 affinity)
1316 {
1317   snat_main_t *sm = &snat_main;
1318   snat_static_mapping_t *m;
1319   snat_session_key_t m_key;
1320   clib_bihash_kv_8_8_t kv, value;
1321   snat_address_t *a = 0;
1322   int i;
1323   nat44_lb_addr_port_t *local;
1324   u32 elt_index, head_index, ses_index;
1325   snat_main_per_thread_data_t *tsm;
1326   snat_user_key_t u_key;
1327   snat_user_t *u;
1328   snat_session_t *s;
1329   dlist_elt_t *head, *elt;
1330   uword *bitmap = 0;
1331
1332   if (!sm->endpoint_dependent)
1333     return VNET_API_ERROR_FEATURE_DISABLED;
1334
1335   m_key.addr = e_addr;
1336   m_key.port = e_port;
1337   m_key.protocol = proto;
1338   m_key.fib_index = 0;
1339   kv.key = m_key.as_u64;
1340   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1341     m = 0;
1342   else
1343     m = pool_elt_at_index (sm->static_mappings, value.value);
1344
1345   if (is_add)
1346     {
1347       if (m)
1348         return VNET_API_ERROR_VALUE_EXIST;
1349
1350       if (vec_len (locals) < 2)
1351         return VNET_API_ERROR_INVALID_VALUE;
1352
1353       /* Find external address in allocated addresses and reserve port for
1354          address and port pair mapping when dynamic translations enabled */
1355       if (!(sm->static_mapping_only || out2in_only))
1356         {
1357           for (i = 0; i < vec_len (sm->addresses); i++)
1358             {
1359               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1360                 {
1361                   a = sm->addresses + i;
1362                   /* External port must be unused */
1363                   switch (proto)
1364                     {
1365 #define _(N, j, n, s) \
1366                     case SNAT_PROTOCOL_##N: \
1367                       if (a->busy_##n##_port_refcounts[e_port]) \
1368                         return VNET_API_ERROR_INVALID_VALUE; \
1369                       ++a->busy_##n##_port_refcounts[e_port]; \
1370                       if (e_port > 1024) \
1371                         { \
1372                           a->busy_##n##_ports++; \
1373                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1374                         } \
1375                       break;
1376                       foreach_snat_protocol
1377 #undef _
1378                     default:
1379                       nat_elog_info ("unknown protocol");
1380                       return VNET_API_ERROR_INVALID_VALUE_2;
1381                     }
1382                   break;
1383                 }
1384             }
1385           /* External address must be allocated */
1386           if (!a)
1387             return VNET_API_ERROR_NO_SUCH_ENTRY;
1388         }
1389
1390       pool_get (sm->static_mappings, m);
1391       clib_memset (m, 0, sizeof (*m));
1392       m->tag = vec_dup (tag);
1393       m->external_addr = e_addr;
1394       m->external_port = e_port;
1395       m->proto = proto;
1396       m->twice_nat = twice_nat;
1397       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1398       if (out2in_only)
1399         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1400       m->affinity = affinity;
1401
1402       if (affinity)
1403         m->affinity_per_service_list_head_index =
1404           nat_affinity_get_per_service_list_head_index ();
1405       else
1406         m->affinity_per_service_list_head_index = ~0;
1407
1408       m_key.addr = m->external_addr;
1409       m_key.port = m->external_port;
1410       m_key.protocol = m->proto;
1411       m_key.fib_index = 0;
1412       kv.key = m_key.as_u64;
1413       kv.value = m - sm->static_mappings;
1414       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1415         {
1416           nat_elog_err ("static_mapping_by_external key add failed");
1417           return VNET_API_ERROR_UNSPECIFIED;
1418         }
1419
1420       m_key.fib_index = m->fib_index;
1421       for (i = 0; i < vec_len (locals); i++)
1422         {
1423           locals[i].fib_index =
1424             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1425                                                locals[i].vrf_id,
1426                                                nat_fib_src_low);
1427           m_key.addr = locals[i].addr;
1428           m_key.fib_index = locals[i].fib_index;
1429           if (!out2in_only)
1430             {
1431               m_key.port = locals[i].port;
1432               kv.key = m_key.as_u64;
1433               kv.value = m - sm->static_mappings;
1434               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1435             }
1436           locals[i].prefix = (i == 0) ? locals[i].probability :
1437             (locals[i - 1].prefix + locals[i].probability);
1438           pool_get (m->locals, local);
1439           *local = locals[i];
1440           if (sm->num_workers > 1)
1441             {
1442               ip4_header_t ip = {
1443                 .src_address = locals[i].addr,
1444               };
1445               bitmap =
1446                 clib_bitmap_set (bitmap,
1447                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1448                                  1);
1449             }
1450         }
1451
1452       /* Assign workers */
1453       if (sm->num_workers > 1)
1454         {
1455           /* *INDENT-OFF* */
1456           clib_bitmap_foreach (i, bitmap,
1457             ({
1458                vec_add1(m->workers, i);
1459             }));
1460           /* *INDENT-ON* */
1461         }
1462     }
1463   else
1464     {
1465       if (!m)
1466         return VNET_API_ERROR_NO_SUCH_ENTRY;
1467
1468       if (!is_lb_static_mapping (m))
1469         return VNET_API_ERROR_INVALID_VALUE;
1470
1471       /* Free external address port */
1472       if (!(sm->static_mapping_only || out2in_only))
1473         {
1474           for (i = 0; i < vec_len (sm->addresses); i++)
1475             {
1476               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1477                 {
1478                   a = sm->addresses + i;
1479                   switch (proto)
1480                     {
1481 #define _(N, j, n, s) \
1482                     case SNAT_PROTOCOL_##N: \
1483                       --a->busy_##n##_port_refcounts[e_port]; \
1484                       if (e_port > 1024) \
1485                         { \
1486                           a->busy_##n##_ports--; \
1487                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1488                         } \
1489                       break;
1490                       foreach_snat_protocol
1491 #undef _
1492                     default:
1493                       nat_elog_info ("unknown protocol");
1494                       return VNET_API_ERROR_INVALID_VALUE_2;
1495                     }
1496                   break;
1497                 }
1498             }
1499         }
1500
1501       m_key.addr = m->external_addr;
1502       m_key.port = m->external_port;
1503       m_key.protocol = m->proto;
1504       m_key.fib_index = 0;
1505       kv.key = m_key.as_u64;
1506       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1507         {
1508           nat_elog_err ("static_mapping_by_external key del failed");
1509           return VNET_API_ERROR_UNSPECIFIED;
1510         }
1511
1512       /* *INDENT-OFF* */
1513       pool_foreach (local, m->locals,
1514       ({
1515           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1516                             nat_fib_src_low);
1517           m_key.addr = local->addr;
1518           if (!out2in_only)
1519             {
1520               m_key.port = local->port;
1521               m_key.fib_index = local->fib_index;
1522               kv.key = m_key.as_u64;
1523               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1524                 {
1525                   nat_elog_err ("static_mapping_by_local key del failed");
1526                   return VNET_API_ERROR_UNSPECIFIED;
1527                 }
1528             }
1529
1530           if (sm->num_workers > 1)
1531             {
1532               ip4_header_t ip = {
1533                 .src_address = local->addr,
1534               };
1535               tsm = vec_elt_at_index (sm->per_thread_data,
1536                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1537             }
1538           else
1539             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1540
1541           /* Delete sessions */
1542           u_key.addr = local->addr;
1543           u_key.fib_index = local->fib_index;
1544           kv.key = u_key.as_u64;
1545           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1546             {
1547               u = pool_elt_at_index (tsm->users, value.value);
1548               if (u->nstaticsessions)
1549                 {
1550                   head_index = u->sessions_per_user_list_head_index;
1551                   head = pool_elt_at_index (tsm->list_pool, head_index);
1552                   elt_index = head->next;
1553                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1554                   ses_index = elt->value;
1555                   while (ses_index != ~0)
1556                     {
1557                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1558                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1559                       ses_index = elt->value;
1560
1561                       if (!(is_lb_session (s)))
1562                         continue;
1563
1564                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1565                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1566                         continue;
1567
1568                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1569                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1570                     }
1571                 }
1572             }
1573       }));
1574       /* *INDENT-ON* */
1575       if (m->affinity)
1576         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1577       pool_free (m->locals);
1578       vec_free (m->tag);
1579       vec_free (m->workers);
1580
1581       pool_put (sm->static_mappings, m);
1582     }
1583
1584   return 0;
1585 }
1586
1587 int
1588 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1589                                        ip4_address_t l_addr, u16 l_port,
1590                                        snat_protocol_t proto, u32 vrf_id,
1591                                        u8 probability, u8 is_add)
1592 {
1593   snat_main_t *sm = &snat_main;
1594   snat_static_mapping_t *m = 0;
1595   snat_session_key_t m_key;
1596   clib_bihash_kv_8_8_t kv, value;
1597   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1598   snat_main_per_thread_data_t *tsm;
1599   snat_user_key_t u_key;
1600   snat_user_t *u;
1601   snat_session_t *s;
1602   dlist_elt_t *head, *elt;
1603   u32 elt_index, head_index, ses_index, *locals = 0;
1604   uword *bitmap = 0;
1605   int i;
1606
1607   if (!sm->endpoint_dependent)
1608     return VNET_API_ERROR_FEATURE_DISABLED;
1609
1610   m_key.addr = e_addr;
1611   m_key.port = e_port;
1612   m_key.protocol = proto;
1613   m_key.fib_index = 0;
1614   kv.key = m_key.as_u64;
1615   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1616     m = pool_elt_at_index (sm->static_mappings, value.value);
1617
1618   if (!m)
1619     return VNET_API_ERROR_NO_SUCH_ENTRY;
1620
1621   if (!is_lb_static_mapping (m))
1622     return VNET_API_ERROR_INVALID_VALUE;
1623
1624   /* *INDENT-OFF* */
1625   pool_foreach (local, m->locals,
1626   ({
1627     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1628         (local->vrf_id == vrf_id))
1629       {
1630         match_local = local;
1631         break;
1632       }
1633   }));
1634   /* *INDENT-ON* */
1635
1636   if (is_add)
1637     {
1638       if (match_local)
1639         return VNET_API_ERROR_VALUE_EXIST;
1640
1641       pool_get (m->locals, local);
1642       clib_memset (local, 0, sizeof (*local));
1643       local->addr.as_u32 = l_addr.as_u32;
1644       local->port = l_port;
1645       local->probability = probability;
1646       local->vrf_id = vrf_id;
1647       local->fib_index =
1648         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1649                                            nat_fib_src_low);
1650
1651       if (!is_out2in_only_static_mapping (m))
1652         {
1653           m_key.addr = l_addr;
1654           m_key.port = l_port;
1655           m_key.fib_index = local->fib_index;
1656           kv.key = m_key.as_u64;
1657           kv.value = m - sm->static_mappings;
1658           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1659             nat_elog_err ("static_mapping_by_local key add failed");
1660         }
1661     }
1662   else
1663     {
1664       if (!match_local)
1665         return VNET_API_ERROR_NO_SUCH_ENTRY;
1666
1667       if (pool_elts (m->locals) < 3)
1668         return VNET_API_ERROR_UNSPECIFIED;
1669
1670       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1671                         nat_fib_src_low);
1672
1673       if (!is_out2in_only_static_mapping (m))
1674         {
1675           m_key.addr = l_addr;
1676           m_key.port = l_port;
1677           m_key.fib_index = match_local->fib_index;
1678           kv.key = m_key.as_u64;
1679           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1680             nat_elog_err ("static_mapping_by_local key del failed");
1681         }
1682
1683       if (sm->num_workers > 1)
1684         {
1685           ip4_header_t ip = {
1686             .src_address = local->addr,
1687           };
1688           tsm = vec_elt_at_index (sm->per_thread_data,
1689                                   sm->worker_in2out_cb (&ip, m->fib_index,
1690                                                         0));
1691         }
1692       else
1693         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1694
1695       /* Delete sessions */
1696       u_key.addr = match_local->addr;
1697       u_key.fib_index = match_local->fib_index;
1698       kv.key = u_key.as_u64;
1699       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1700         {
1701           u = pool_elt_at_index (tsm->users, value.value);
1702           if (u->nstaticsessions)
1703             {
1704               head_index = u->sessions_per_user_list_head_index;
1705               head = pool_elt_at_index (tsm->list_pool, head_index);
1706               elt_index = head->next;
1707               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1708               ses_index = elt->value;
1709               while (ses_index != ~0)
1710                 {
1711                   s = pool_elt_at_index (tsm->sessions, ses_index);
1712                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1713                   ses_index = elt->value;
1714
1715                   if (!(is_lb_session (s)))
1716                     continue;
1717
1718                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1719                       (clib_net_to_host_u16 (s->in2out.port) !=
1720                        match_local->port))
1721                     continue;
1722
1723                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1724                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1725                 }
1726             }
1727         }
1728
1729       pool_put (m->locals, match_local);
1730     }
1731
1732   vec_free (m->workers);
1733
1734   /* *INDENT-OFF* */
1735   pool_foreach (local, m->locals,
1736   ({
1737     vec_add1 (locals, local - m->locals);
1738     if (sm->num_workers > 1)
1739       {
1740         ip4_header_t ip;
1741         ip.src_address.as_u32 = local->addr.as_u32,
1742         bitmap = clib_bitmap_set (bitmap,
1743                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1744                                   1);
1745       }
1746   }));
1747   /* *INDENT-ON* */
1748
1749   ASSERT (vec_len (locals) > 1);
1750
1751   local = pool_elt_at_index (m->locals, locals[0]);
1752   local->prefix = local->probability;
1753   for (i = 1; i < vec_len (locals); i++)
1754     {
1755       local = pool_elt_at_index (m->locals, locals[i]);
1756       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1757       local->prefix = local->probability + prev_local->prefix;
1758     }
1759
1760   /* Assign workers */
1761   if (sm->num_workers > 1)
1762     {
1763       /* *INDENT-OFF* */
1764       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1765       /* *INDENT-ON* */
1766     }
1767
1768   return 0;
1769 }
1770
1771 int
1772 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1773                   u8 twice_nat)
1774 {
1775   snat_address_t *a = 0;
1776   snat_session_t *ses;
1777   u32 *ses_to_be_removed = 0, *ses_index;
1778   snat_main_per_thread_data_t *tsm;
1779   snat_static_mapping_t *m;
1780   snat_interface_t *interface;
1781   int i;
1782   snat_address_t *addresses =
1783     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1784
1785   /* Find SNAT address */
1786   for (i = 0; i < vec_len (addresses); i++)
1787     {
1788       if (addresses[i].addr.as_u32 == addr.as_u32)
1789         {
1790           a = addresses + i;
1791           break;
1792         }
1793     }
1794   if (!a)
1795     return VNET_API_ERROR_NO_SUCH_ENTRY;
1796
1797   if (delete_sm)
1798     {
1799       /* *INDENT-OFF* */
1800       pool_foreach (m, sm->static_mappings,
1801       ({
1802           if (m->external_addr.as_u32 == addr.as_u32)
1803             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1804                                             m->local_port, m->external_port,
1805                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1806                                             m->proto, 0, m->twice_nat,
1807                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1808       }));
1809       /* *INDENT-ON* */
1810     }
1811   else
1812     {
1813       /* Check if address is used in some static mapping */
1814       if (is_snat_address_used_in_static_mapping (sm, addr))
1815         {
1816           nat_elog_notice ("address used in static mapping");
1817           return VNET_API_ERROR_UNSPECIFIED;
1818         }
1819     }
1820
1821   if (a->fib_index != ~0)
1822     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1823
1824   /* Delete sessions using address */
1825   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1826     {
1827       /* *INDENT-OFF* */
1828       vec_foreach (tsm, sm->per_thread_data)
1829         {
1830           pool_foreach (ses, tsm->sessions, ({
1831             if (ses->out2in.addr.as_u32 == addr.as_u32)
1832               {
1833                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1834                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1835               }
1836           }));
1837
1838           vec_foreach (ses_index, ses_to_be_removed)
1839             {
1840               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1841               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1842             }
1843
1844           vec_free (ses_to_be_removed);
1845         }
1846       /* *INDENT-ON* */
1847     }
1848
1849 #define _(N, i, n, s) \
1850   vec_free (a->busy_##n##_ports_per_thread);
1851   foreach_snat_protocol
1852 #undef _
1853     if (twice_nat)
1854     {
1855       vec_del1 (sm->twice_nat_addresses, i);
1856       return 0;
1857     }
1858   else
1859     vec_del1 (sm->addresses, i);
1860
1861   /* Delete external address from FIB */
1862   /* *INDENT-OFF* */
1863   pool_foreach (interface, sm->interfaces,
1864   ({
1865     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1866       continue;
1867
1868     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1869     break;
1870   }));
1871   pool_foreach (interface, sm->output_feature_interfaces,
1872   ({
1873     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1874       continue;
1875
1876     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1877     break;
1878   }));
1879   /* *INDENT-ON* */
1880
1881   return 0;
1882 }
1883
1884 int
1885 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1886 {
1887   snat_main_t *sm = &snat_main;
1888   snat_interface_t *i;
1889   const char *feature_name, *del_feature_name;
1890   snat_address_t *ap;
1891   snat_static_mapping_t *m;
1892   snat_det_map_t *dm;
1893   nat_outside_fib_t *outside_fib;
1894   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1895                                                        sw_if_index);
1896
1897   if (sm->out2in_dpo && !is_inside)
1898     return VNET_API_ERROR_UNSUPPORTED;
1899
1900   /* *INDENT-OFF* */
1901   pool_foreach (i, sm->output_feature_interfaces,
1902   ({
1903     if (i->sw_if_index == sw_if_index)
1904       return VNET_API_ERROR_VALUE_EXIST;
1905   }));
1906   /* *INDENT-ON* */
1907
1908   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1909     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1910   else
1911     {
1912       if (sm->num_workers > 1 && !sm->deterministic)
1913         feature_name =
1914           is_inside ? "nat44-in2out-worker-handoff" :
1915           "nat44-out2in-worker-handoff";
1916       else if (sm->deterministic)
1917         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1918       else if (sm->endpoint_dependent)
1919         {
1920           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1921         }
1922       else
1923         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1924     }
1925
1926   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1927     sm->fq_in2out_index =
1928       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1929
1930   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1931     sm->fq_out2in_index =
1932       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1933
1934   if (!is_inside)
1935     {
1936       /* *INDENT-OFF* */
1937       vec_foreach (outside_fib, sm->outside_fibs)
1938         {
1939           if (outside_fib->fib_index == fib_index)
1940             {
1941               if (is_del)
1942                 {
1943                   outside_fib->refcount--;
1944                   if (!outside_fib->refcount)
1945                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1946                 }
1947               else
1948                 outside_fib->refcount++;
1949               goto feature_set;
1950             }
1951         }
1952       /* *INDENT-ON* */
1953       if (!is_del)
1954         {
1955           vec_add2 (sm->outside_fibs, outside_fib, 1);
1956           outside_fib->refcount = 1;
1957           outside_fib->fib_index = fib_index;
1958         }
1959     }
1960 feature_set:
1961   /* *INDENT-OFF* */
1962   pool_foreach (i, sm->interfaces,
1963   ({
1964     if (i->sw_if_index == sw_if_index)
1965       {
1966         if (is_del)
1967           {
1968             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1969               {
1970                 if (is_inside)
1971                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1972                 else
1973                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1974
1975                 if (sm->num_workers > 1 && !sm->deterministic)
1976                   {
1977                     del_feature_name = "nat44-handoff-classify";
1978                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1979                                                  "nat44-out2in-worker-handoff";
1980                   }
1981                 else if (sm->deterministic)
1982                   {
1983                     del_feature_name = "nat44-det-classify";
1984                     feature_name = !is_inside ?  "nat44-det-in2out" :
1985                                                  "nat44-det-out2in";
1986                   }
1987                 else if (sm->endpoint_dependent)
1988                   {
1989                     del_feature_name = "nat44-ed-classify";
1990                     feature_name = !is_inside ?  "nat-pre-in2out" :
1991                                                  "nat-pre-out2in";
1992                   }
1993                 else
1994                   {
1995                     del_feature_name = "nat44-classify";
1996                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1997                   }
1998
1999                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2000                 if (rv)
2001                   return rv;
2002                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2003                                              sw_if_index, 0, 0, 0);
2004                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2005                                              sw_if_index, 1, 0, 0);
2006                 if (!is_inside)
2007                   {
2008                     if (sm->endpoint_dependent)
2009                       vnet_feature_enable_disable ("ip4-local",
2010                                                    "nat44-ed-hairpinning",
2011                                                    sw_if_index, 1, 0, 0);
2012                     else if (!sm->deterministic)
2013                       vnet_feature_enable_disable ("ip4-local",
2014                                                    "nat44-hairpinning",
2015                                                    sw_if_index, 1, 0, 0);
2016                   }
2017               }
2018             else
2019               {
2020                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2021                 if (rv)
2022                   return rv;
2023                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2024                                              sw_if_index, 0, 0, 0);
2025                 pool_put (sm->interfaces, i);
2026                 if (is_inside)
2027                   {
2028                     if (sm->endpoint_dependent)
2029                       vnet_feature_enable_disable ("ip4-local",
2030                                                    "nat44-ed-hairpinning",
2031                                                    sw_if_index, 0, 0, 0);
2032                     else if (!sm->deterministic)
2033                       vnet_feature_enable_disable ("ip4-local",
2034                                                    "nat44-hairpinning",
2035                                                    sw_if_index, 0, 0, 0);
2036                   }
2037               }
2038           }
2039         else
2040           {
2041             if ((nat_interface_is_inside(i) && is_inside) ||
2042                 (nat_interface_is_outside(i) && !is_inside))
2043               return 0;
2044
2045             if (sm->num_workers > 1 && !sm->deterministic)
2046               {
2047                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2048                                                  "nat44-out2in-worker-handoff";
2049                 feature_name = "nat44-handoff-classify";
2050               }
2051             else if (sm->deterministic)
2052               {
2053                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
2054                                                  "nat44-det-out2in";
2055                 feature_name = "nat44-det-classify";
2056               }
2057             else if (sm->endpoint_dependent)
2058               {
2059                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2060                                                  "nat-pre-out2in";
2061
2062                 feature_name = "nat44-ed-classify";
2063               }
2064             else
2065               {
2066                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2067                 feature_name = "nat44-classify";
2068               }
2069
2070             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2071             if (rv)
2072               return rv;
2073             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2074                                          sw_if_index, 0, 0, 0);
2075             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2076                                          sw_if_index, 1, 0, 0);
2077             if (!is_inside)
2078               {
2079                 if (sm->endpoint_dependent)
2080                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2081                                                sw_if_index, 0, 0, 0);
2082                 else if (!sm->deterministic)
2083                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2084                                                sw_if_index, 0, 0, 0);
2085               }
2086             goto set_flags;
2087           }
2088
2089         goto fib;
2090       }
2091   }));
2092   /* *INDENT-ON* */
2093
2094   if (is_del)
2095     return VNET_API_ERROR_NO_SUCH_ENTRY;
2096
2097   pool_get (sm->interfaces, i);
2098   i->sw_if_index = sw_if_index;
2099   i->flags = 0;
2100   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2101                                0);
2102
2103   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2104   if (rv)
2105     return rv;
2106
2107   if (is_inside && !sm->out2in_dpo)
2108     {
2109       if (sm->endpoint_dependent)
2110         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2111                                      sw_if_index, 1, 0, 0);
2112       else if (!sm->deterministic)
2113         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2114                                      sw_if_index, 1, 0, 0);
2115     }
2116
2117 set_flags:
2118   if (is_inside)
2119     {
2120       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2121       return 0;
2122     }
2123   else
2124     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2125
2126   /* Add/delete external addresses to FIB */
2127 fib:
2128   /* *INDENT-OFF* */
2129   vec_foreach (ap, sm->addresses)
2130     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2131
2132   pool_foreach (m, sm->static_mappings,
2133   ({
2134     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2135       continue;
2136
2137     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2138   }));
2139
2140   pool_foreach (dm, sm->det_maps,
2141   ({
2142     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2143   }));
2144   /* *INDENT-ON* */
2145
2146   return 0;
2147 }
2148
2149 int
2150 snat_interface_add_del_output_feature (u32 sw_if_index,
2151                                        u8 is_inside, int is_del)
2152 {
2153   snat_main_t *sm = &snat_main;
2154   snat_interface_t *i;
2155   snat_address_t *ap;
2156   snat_static_mapping_t *m;
2157   nat_outside_fib_t *outside_fib;
2158   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2159                                                        sw_if_index);
2160
2161
2162   if (sm->deterministic ||
2163       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2164     return VNET_API_ERROR_UNSUPPORTED;
2165
2166   /* *INDENT-OFF* */
2167   pool_foreach (i, sm->interfaces,
2168   ({
2169     if (i->sw_if_index == sw_if_index)
2170       return VNET_API_ERROR_VALUE_EXIST;
2171   }));
2172   /* *INDENT-ON* */
2173
2174   if (!is_inside)
2175     {
2176       /* *INDENT-OFF* */
2177       vec_foreach (outside_fib, sm->outside_fibs)
2178         {
2179           if (outside_fib->fib_index == fib_index)
2180             {
2181               if (is_del)
2182                 {
2183                   outside_fib->refcount--;
2184                   if (!outside_fib->refcount)
2185                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2186                 }
2187               else
2188                 outside_fib->refcount++;
2189               goto feature_set;
2190             }
2191         }
2192       /* *INDENT-ON* */
2193       if (!is_del)
2194         {
2195           vec_add2 (sm->outside_fibs, outside_fib, 1);
2196           outside_fib->refcount = 1;
2197           outside_fib->fib_index = fib_index;
2198         }
2199     }
2200
2201 feature_set:
2202   if (is_inside)
2203     {
2204       if (sm->endpoint_dependent)
2205         {
2206           int rv =
2207             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2208           if (rv)
2209             return rv;
2210           rv =
2211             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2212                                                             !is_del);
2213           if (rv)
2214             return rv;
2215           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2216                                        sw_if_index, !is_del, 0, 0);
2217           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2218                                        sw_if_index, !is_del, 0, 0);
2219         }
2220       else
2221         {
2222           int rv =
2223             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2224           if (rv)
2225             return rv;
2226           rv =
2227             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2228                                                             !is_del);
2229           if (rv)
2230             return rv;
2231           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2232                                        sw_if_index, !is_del, 0, 0);
2233           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2234                                        sw_if_index, !is_del, 0, 0);
2235         }
2236       goto fq;
2237     }
2238
2239   if (sm->num_workers > 1)
2240     {
2241       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2242       if (rv)
2243         return rv;
2244       rv =
2245         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2246       if (rv)
2247         return rv;
2248       vnet_feature_enable_disable ("ip4-unicast",
2249                                    "nat44-out2in-worker-handoff",
2250                                    sw_if_index, !is_del, 0, 0);
2251       vnet_feature_enable_disable ("ip4-output",
2252                                    "nat44-in2out-output-worker-handoff",
2253                                    sw_if_index, !is_del, 0, 0);
2254     }
2255   else
2256     {
2257       if (sm->endpoint_dependent)
2258         {
2259           int rv =
2260             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2261           if (rv)
2262             return rv;
2263           rv =
2264             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2265                                                             !is_del);
2266           if (rv)
2267             return rv;
2268           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2269                                        sw_if_index, !is_del, 0, 0);
2270           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2271                                        sw_if_index, !is_del, 0, 0);
2272         }
2273       else
2274         {
2275           int rv =
2276             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2277           if (rv)
2278             return rv;
2279           rv =
2280             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2281                                                             !is_del);
2282           if (rv)
2283             return rv;
2284           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2285                                        sw_if_index, !is_del, 0, 0);
2286           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2287                                        sw_if_index, !is_del, 0, 0);
2288         }
2289     }
2290
2291 fq:
2292   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2293     sm->fq_in2out_output_index =
2294       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2295
2296   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2297     sm->fq_out2in_index =
2298       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2299
2300   /* *INDENT-OFF* */
2301   pool_foreach (i, sm->output_feature_interfaces,
2302   ({
2303     if (i->sw_if_index == sw_if_index)
2304       {
2305         if (is_del)
2306           pool_put (sm->output_feature_interfaces, i);
2307         else
2308           return VNET_API_ERROR_VALUE_EXIST;
2309
2310         goto fib;
2311       }
2312   }));
2313   /* *INDENT-ON* */
2314
2315   if (is_del)
2316     return VNET_API_ERROR_NO_SUCH_ENTRY;
2317
2318   pool_get (sm->output_feature_interfaces, i);
2319   i->sw_if_index = sw_if_index;
2320   i->flags = 0;
2321   if (is_inside)
2322     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2323   else
2324     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2325
2326   /* Add/delete external addresses to FIB */
2327 fib:
2328   if (is_inside)
2329     return 0;
2330
2331   /* *INDENT-OFF* */
2332   vec_foreach (ap, sm->addresses)
2333     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2334
2335   pool_foreach (m, sm->static_mappings,
2336   ({
2337     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2338       continue;
2339
2340     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2341   }));
2342   /* *INDENT-ON* */
2343
2344   return 0;
2345 }
2346
2347 int
2348 snat_set_workers (uword * bitmap)
2349 {
2350   snat_main_t *sm = &snat_main;
2351   int i, j = 0;
2352
2353   if (sm->num_workers < 2)
2354     return VNET_API_ERROR_FEATURE_DISABLED;
2355
2356   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2357     return VNET_API_ERROR_INVALID_WORKER;
2358
2359   vec_free (sm->workers);
2360   /* *INDENT-OFF* */
2361   clib_bitmap_foreach (i, bitmap,
2362     ({
2363       vec_add1(sm->workers, i);
2364       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2365       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2366       j++;
2367     }));
2368   /* *INDENT-ON* */
2369
2370   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2371   sm->num_snat_thread = _vec_len (sm->workers);
2372
2373   return 0;
2374 }
2375
2376 static void
2377 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2378                          u32 old_fib_index)
2379 {
2380   snat_main_t *sm = &snat_main;
2381   nat_outside_fib_t *outside_fib;
2382   snat_interface_t *i;
2383   u8 is_add = 1;
2384   u8 match = 0;
2385
2386   if (new_fib_index == old_fib_index)
2387     return;
2388
2389   if (!vec_len (sm->outside_fibs))
2390     return;
2391
2392   /* *INDENT-OFF* */
2393   pool_foreach (i, sm->interfaces,
2394     ({
2395       if (i->sw_if_index == sw_if_index)
2396         {
2397           if (!(nat_interface_is_outside (i)))
2398             return;
2399           match = 1;
2400         }
2401     }));
2402
2403   pool_foreach (i, sm->output_feature_interfaces,
2404     ({
2405       if (i->sw_if_index == sw_if_index)
2406         {
2407           if (!(nat_interface_is_outside (i)))
2408             return;
2409           match = 1;
2410         }
2411     }));
2412   /* *INDENT-ON* */
2413
2414   if (!match)
2415     return;
2416
2417   vec_foreach (outside_fib, sm->outside_fibs)
2418   {
2419     if (outside_fib->fib_index == old_fib_index)
2420       {
2421         outside_fib->refcount--;
2422         if (!outside_fib->refcount)
2423           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2424         break;
2425       }
2426   }
2427
2428   vec_foreach (outside_fib, sm->outside_fibs)
2429   {
2430     if (outside_fib->fib_index == new_fib_index)
2431       {
2432         outside_fib->refcount++;
2433         is_add = 0;
2434         break;
2435       }
2436   }
2437
2438   if (is_add)
2439     {
2440       vec_add2 (sm->outside_fibs, outside_fib, 1);
2441       outside_fib->refcount = 1;
2442       outside_fib->fib_index = new_fib_index;
2443     }
2444 }
2445
2446 static void
2447 snat_ip4_table_bind (ip4_main_t * im,
2448                      uword opaque,
2449                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2450 {
2451   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2452 }
2453
2454 static void
2455 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2456                                        uword opaque,
2457                                        u32 sw_if_index,
2458                                        ip4_address_t * address,
2459                                        u32 address_length,
2460                                        u32 if_address_index, u32 is_delete);
2461
2462 static void
2463 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2464                                  uword opaque,
2465                                  u32 sw_if_index,
2466                                  ip4_address_t * address,
2467                                  u32 address_length,
2468                                  u32 if_address_index, u32 is_delete);
2469
2470 static int
2471 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2472                                  u32 fib_index,
2473                                  u32 thread_index,
2474                                  snat_session_key_t * k,
2475                                  u16 port_per_thread, u32 snat_thread_index);
2476
2477 static clib_error_t *
2478 snat_init (vlib_main_t * vm)
2479 {
2480   snat_main_t *sm = &snat_main;
2481   clib_error_t *error = 0;
2482   ip4_main_t *im = &ip4_main;
2483   ip_lookup_main_t *lm = &im->lookup_main;
2484   uword *p;
2485   vlib_thread_registration_t *tr;
2486   vlib_thread_main_t *tm = vlib_get_thread_main ();
2487   uword *bitmap = 0;
2488   u32 i;
2489   ip4_add_del_interface_address_callback_t cb4;
2490   vlib_node_t *node;
2491
2492   sm->vlib_main = vm;
2493   sm->vnet_main = vnet_get_main ();
2494   sm->ip4_main = im;
2495   sm->ip4_lookup_main = lm;
2496   sm->api_main = vlibapi_get_main ();
2497   sm->first_worker_index = 0;
2498   sm->num_workers = 0;
2499   sm->num_snat_thread = 1;
2500   sm->workers = 0;
2501   sm->port_per_thread = 0xffff - 1024;
2502   sm->fq_in2out_index = ~0;
2503   sm->fq_in2out_output_index = ~0;
2504   sm->fq_out2in_index = ~0;
2505
2506   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2507   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2508   sm->forwarding_enabled = 0;
2509   sm->log_class = vlib_log_register_class ("nat", 0);
2510   sm->log_level = SNAT_LOG_ERROR;
2511   sm->mss_clamping = 0;
2512
2513   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2514   sm->error_node_index = node->index;
2515
2516   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2517   sm->pre_in2out_node_index = node->index;
2518   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2519   sm->pre_out2in_node_index = node->index;
2520
2521   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2522   sm->pre_in2out_node_index = node->index;
2523
2524   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2525   sm->pre_out2in_node_index = node->index;
2526
2527   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2528   sm->in2out_node_index = node->index;
2529   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2530   sm->in2out_output_node_index = node->index;
2531   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2532   sm->in2out_fast_node_index = node->index;
2533   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2534   sm->in2out_slowpath_node_index = node->index;
2535   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2536   sm->in2out_slowpath_output_node_index = node->index;
2537
2538   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2539   sm->ed_in2out_node_index = node->index;
2540   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2541   sm->ed_in2out_slowpath_node_index = node->index;
2542
2543   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2544   sm->out2in_node_index = node->index;
2545   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2546   sm->out2in_fast_node_index = node->index;
2547
2548   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2549   sm->ed_out2in_node_index = node->index;
2550   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2551   sm->ed_out2in_slowpath_node_index = node->index;
2552
2553   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2554   sm->det_in2out_node_index = node->index;
2555   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2556   sm->det_out2in_node_index = node->index;
2557
2558   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2559   sm->hairpinning_node_index = node->index;
2560   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2561   sm->hairpin_dst_node_index = node->index;
2562   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2563   sm->hairpin_src_node_index = node->index;
2564   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2565   sm->ed_hairpinning_node_index = node->index;
2566   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2567   sm->ed_hairpin_dst_node_index = node->index;
2568   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2569   sm->ed_hairpin_src_node_index = node->index;
2570
2571   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2572   if (p)
2573     {
2574       tr = (vlib_thread_registration_t *) p[0];
2575       if (tr)
2576         {
2577           sm->num_workers = tr->count;
2578           sm->first_worker_index = tr->first_index;
2579         }
2580     }
2581
2582   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2583
2584   /* Use all available workers by default */
2585   if (sm->num_workers > 1)
2586     {
2587       for (i = 0; i < sm->num_workers; i++)
2588         bitmap = clib_bitmap_set (bitmap, i, 1);
2589       snat_set_workers (bitmap);
2590       clib_bitmap_free (bitmap);
2591     }
2592   else
2593     {
2594       sm->per_thread_data[0].snat_thread_index = 0;
2595     }
2596
2597   error = snat_api_init (vm, sm);
2598   if (error)
2599     return error;
2600
2601   /* Set up the interface address add/del callback */
2602   cb4.function = snat_ip4_add_del_interface_address_cb;
2603   cb4.function_opaque = 0;
2604
2605   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2606
2607   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2608   cb4.function_opaque = 0;
2609
2610   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2611
2612   nat_dpo_module_init ();
2613
2614   /* Init counters */
2615   sm->total_users.name = "total-users";
2616   sm->total_users.stat_segment_name = "/nat44/total-users";
2617   vlib_validate_simple_counter (&sm->total_users, 0);
2618   vlib_zero_simple_counter (&sm->total_users, 0);
2619   sm->total_sessions.name = "total-sessions";
2620   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2621   vlib_validate_simple_counter (&sm->total_sessions, 0);
2622   vlib_zero_simple_counter (&sm->total_sessions, 0);
2623
2624   /* Init IPFIX logging */
2625   snat_ipfix_logging_init (vm);
2626
2627   /* Init NAT64 */
2628   error = nat64_init (vm);
2629   if (error)
2630     return error;
2631
2632   nat66_init (vm);
2633
2634   ip4_table_bind_callback_t cbt4 = {
2635     .function = snat_ip4_table_bind,
2636   };
2637   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2638
2639   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2640                                         FIB_SOURCE_PRIORITY_HI,
2641                                         FIB_SOURCE_BH_SIMPLE);
2642   nat_fib_src_low = fib_source_allocate ("nat-low",
2643                                          FIB_SOURCE_PRIORITY_LOW,
2644                                          FIB_SOURCE_BH_SIMPLE);
2645
2646   return error;
2647 }
2648
2649 VLIB_INIT_FUNCTION (snat_init);
2650
2651 void
2652 snat_free_outside_address_and_port (snat_address_t * addresses,
2653                                     u32 thread_index, snat_session_key_t * k)
2654 {
2655   snat_address_t *a;
2656   u32 address_index;
2657   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2658
2659   for (address_index = 0; address_index < vec_len (addresses);
2660        address_index++)
2661     {
2662       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2663         break;
2664     }
2665
2666   ASSERT (address_index < vec_len (addresses));
2667
2668   a = addresses + address_index;
2669
2670   switch (k->protocol)
2671     {
2672 #define _(N, i, n, s) \
2673     case SNAT_PROTOCOL_##N: \
2674       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2675       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2676       a->busy_##n##_ports--; \
2677       a->busy_##n##_ports_per_thread[thread_index]--; \
2678       break;
2679       foreach_snat_protocol
2680 #undef _
2681     default:
2682       nat_elog_info ("unknown protocol");
2683       return;
2684     }
2685 }
2686
2687 static int
2688 nat_set_outside_address_and_port (snat_address_t * addresses,
2689                                   u32 thread_index, snat_session_key_t * k)
2690 {
2691   snat_address_t *a = 0;
2692   u32 address_index;
2693   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2694
2695   for (address_index = 0; address_index < vec_len (addresses);
2696        address_index++)
2697     {
2698       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2699         continue;
2700
2701       a = addresses + address_index;
2702       switch (k->protocol)
2703         {
2704 #define _(N, j, n, s) \
2705         case SNAT_PROTOCOL_##N: \
2706           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2707             return VNET_API_ERROR_INSTANCE_IN_USE; \
2708           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2709           a->busy_##n##_ports_per_thread[thread_index]++; \
2710           a->busy_##n##_ports++; \
2711           return 0;
2712           foreach_snat_protocol
2713 #undef _
2714         default:
2715           nat_elog_info ("unknown protocol");
2716           return 1;
2717         }
2718     }
2719
2720   return VNET_API_ERROR_NO_SUCH_ENTRY;
2721 }
2722
2723 int
2724 snat_static_mapping_match (snat_main_t * sm,
2725                            snat_session_key_t match,
2726                            snat_session_key_t * mapping,
2727                            u8 by_external,
2728                            u8 * is_addr_only,
2729                            twice_nat_type_t * twice_nat,
2730                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2731                            u8 * is_identity_nat)
2732 {
2733   clib_bihash_kv_8_8_t kv, value;
2734   snat_static_mapping_t *m;
2735   snat_session_key_t m_key;
2736   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2737   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2738   u8 backend_index;
2739   nat44_lb_addr_port_t *local;
2740
2741   m_key.fib_index = match.fib_index;
2742   if (by_external)
2743     {
2744       mapping_hash = &sm->static_mapping_by_external;
2745       m_key.fib_index = 0;
2746     }
2747
2748   m_key.addr = match.addr;
2749   m_key.port = clib_net_to_host_u16 (match.port);
2750   m_key.protocol = match.protocol;
2751
2752   kv.key = m_key.as_u64;
2753
2754   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2755     {
2756       /* Try address only mapping */
2757       m_key.port = 0;
2758       m_key.protocol = 0;
2759       kv.key = m_key.as_u64;
2760       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2761         return 1;
2762     }
2763
2764   m = pool_elt_at_index (sm->static_mappings, value.value);
2765
2766   if (by_external)
2767     {
2768       if (is_lb_static_mapping (m))
2769         {
2770           if (PREDICT_FALSE (lb != 0))
2771             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2772           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2773                                                           match.addr,
2774                                                           match.protocol,
2775                                                           match.port,
2776                                                           &backend_index))
2777             {
2778               local = pool_elt_at_index (m->locals, backend_index);
2779               mapping->addr = local->addr;
2780               mapping->port = clib_host_to_net_u16 (local->port);
2781               mapping->fib_index = local->fib_index;
2782               goto end;
2783             }
2784           // pick locals matching this worker
2785           if (PREDICT_FALSE (sm->num_workers > 1))
2786             {
2787               u32 thread_index = vlib_get_thread_index ();
2788               /* *INDENT-OFF* */
2789               pool_foreach_index (i, m->locals,
2790               ({
2791                 local = pool_elt_at_index (m->locals, i);
2792
2793                 ip4_header_t ip = {
2794                   .src_address = local->addr,
2795                 };
2796
2797                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2798                     thread_index)
2799                   {
2800                     vec_add1 (tmp, i);
2801                   }
2802               }));
2803               /* *INDENT-ON* */
2804               ASSERT (vec_len (tmp) != 0);
2805             }
2806           else
2807             {
2808               /* *INDENT-OFF* */
2809               pool_foreach_index (i, m->locals,
2810               ({
2811                 vec_add1 (tmp, i);
2812               }));
2813               /* *INDENT-ON* */
2814             }
2815           hi = vec_len (tmp) - 1;
2816           local = pool_elt_at_index (m->locals, tmp[hi]);
2817           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2818           while (lo < hi)
2819             {
2820               mid = ((hi - lo) >> 1) + lo;
2821               local = pool_elt_at_index (m->locals, tmp[mid]);
2822               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2823             }
2824           local = pool_elt_at_index (m->locals, tmp[lo]);
2825           if (!(local->prefix >= rand))
2826             return 1;
2827           mapping->addr = local->addr;
2828           mapping->port = clib_host_to_net_u16 (local->port);
2829           mapping->fib_index = local->fib_index;
2830           if (m->affinity)
2831             {
2832               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2833                                                 match.protocol, match.port,
2834                                                 tmp[lo], m->affinity,
2835                                                 m->affinity_per_service_list_head_index))
2836                 nat_elog_info ("create affinity record failed");
2837             }
2838           vec_free (tmp);
2839         }
2840       else
2841         {
2842           if (PREDICT_FALSE (lb != 0))
2843             *lb = NO_LB_NAT;
2844           mapping->fib_index = m->fib_index;
2845           mapping->addr = m->local_addr;
2846           /* Address only mapping doesn't change port */
2847           mapping->port = is_addr_only_static_mapping (m) ? match.port
2848             : clib_host_to_net_u16 (m->local_port);
2849         }
2850       mapping->protocol = m->proto;
2851     }
2852   else
2853     {
2854       mapping->addr = m->external_addr;
2855       /* Address only mapping doesn't change port */
2856       mapping->port = is_addr_only_static_mapping (m) ? match.port
2857         : clib_host_to_net_u16 (m->external_port);
2858       mapping->fib_index = sm->outside_fib_index;
2859     }
2860
2861 end:
2862   if (PREDICT_FALSE (is_addr_only != 0))
2863     *is_addr_only = is_addr_only_static_mapping (m);
2864
2865   if (PREDICT_FALSE (twice_nat != 0))
2866     *twice_nat = m->twice_nat;
2867
2868   if (PREDICT_FALSE (is_identity_nat != 0))
2869     *is_identity_nat = is_identity_static_mapping (m);
2870
2871   return 0;
2872 }
2873
2874 static_always_inline u16
2875 snat_random_port (u16 min, u16 max)
2876 {
2877   snat_main_t *sm = &snat_main;
2878   return min + random_u32 (&sm->random_seed) /
2879     (random_u32_max () / (max - min + 1) + 1);
2880 }
2881
2882 int
2883 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2884                                      u32 fib_index,
2885                                      u32 thread_index,
2886                                      snat_session_key_t * k,
2887                                      u16 port_per_thread,
2888                                      u32 snat_thread_index)
2889 {
2890   snat_main_t *sm = &snat_main;
2891
2892   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2893                                   port_per_thread, snat_thread_index);
2894 }
2895
2896 static int
2897 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2898                                  u32 fib_index,
2899                                  u32 thread_index,
2900                                  snat_session_key_t * k,
2901                                  u16 port_per_thread, u32 snat_thread_index)
2902 {
2903   int i;
2904   snat_address_t *a, *ga = 0;
2905   u32 portnum;
2906
2907   for (i = 0; i < vec_len (addresses); i++)
2908     {
2909       a = addresses + i;
2910       switch (k->protocol)
2911         {
2912 #define _(N, j, n, s) \
2913         case SNAT_PROTOCOL_##N: \
2914           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2915             { \
2916               if (a->fib_index == fib_index) \
2917                 { \
2918                   while (1) \
2919                     { \
2920                       portnum = (port_per_thread * \
2921                         snat_thread_index) + \
2922                         snat_random_port(1, port_per_thread) + 1024; \
2923                       if (a->busy_##n##_port_refcounts[portnum]) \
2924                         continue; \
2925                       --a->busy_##n##_port_refcounts[portnum]; \
2926                       a->busy_##n##_ports_per_thread[thread_index]++; \
2927                       a->busy_##n##_ports++; \
2928                       k->addr = a->addr; \
2929                       k->port = clib_host_to_net_u16(portnum); \
2930                       return 0; \
2931                     } \
2932                 } \
2933               else if (a->fib_index == ~0) \
2934                 { \
2935                   ga = a; \
2936                 } \
2937             } \
2938           break;
2939           foreach_snat_protocol
2940 #undef _
2941         default:
2942           nat_elog_info ("unknown protocol");
2943           return 1;
2944         }
2945
2946     }
2947
2948   if (ga)
2949     {
2950       a = ga;
2951       switch (k->protocol)
2952         {
2953 #define _(N, j, n, s) \
2954         case SNAT_PROTOCOL_##N: \
2955           while (1) \
2956             { \
2957               portnum = (port_per_thread * \
2958                 snat_thread_index) + \
2959                 snat_random_port(1, port_per_thread) + 1024; \
2960               if (a->busy_##n##_port_refcounts[portnum]) \
2961                 continue; \
2962               ++a->busy_##n##_port_refcounts[portnum]; \
2963               a->busy_##n##_ports_per_thread[thread_index]++; \
2964               a->busy_##n##_ports++; \
2965               k->addr = a->addr; \
2966               k->port = clib_host_to_net_u16(portnum); \
2967               return 0; \
2968             }
2969           break;
2970           foreach_snat_protocol
2971 #undef _
2972         default:
2973           nat_elog_info ("unknown protocol");
2974           return 1;
2975         }
2976     }
2977
2978   /* Totally out of translations to use... */
2979   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2980   return 1;
2981 }
2982
2983 static int
2984 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2985                               u32 fib_index,
2986                               u32 thread_index,
2987                               snat_session_key_t * k,
2988                               u16 port_per_thread, u32 snat_thread_index)
2989 {
2990   snat_main_t *sm = &snat_main;
2991   snat_address_t *a = addresses;
2992   u16 m, ports, portnum, A, j;
2993   m = 16 - (sm->psid_offset + sm->psid_length);
2994   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2995
2996   if (!vec_len (addresses))
2997     goto exhausted;
2998
2999   switch (k->protocol)
3000     {
3001 #define _(N, i, n, s) \
3002     case SNAT_PROTOCOL_##N: \
3003       if (a->busy_##n##_ports < ports) \
3004         { \
3005           while (1) \
3006             { \
3007               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3008               j = snat_random_port(0, pow2_mask(m)); \
3009               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3010               if (a->busy_##n##_port_refcounts[portnum]) \
3011                 continue; \
3012               ++a->busy_##n##_port_refcounts[portnum]; \
3013               a->busy_##n##_ports++; \
3014               k->addr = a->addr; \
3015               k->port = clib_host_to_net_u16 (portnum); \
3016               return 0; \
3017             } \
3018         } \
3019       break;
3020       foreach_snat_protocol
3021 #undef _
3022     default:
3023       nat_elog_info ("unknown protocol");
3024       return 1;
3025     }
3026
3027 exhausted:
3028   /* Totally out of translations to use... */
3029   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3030   return 1;
3031 }
3032
3033 static int
3034 nat_alloc_addr_and_port_range (snat_address_t * addresses,
3035                                u32 fib_index,
3036                                u32 thread_index,
3037                                snat_session_key_t * k,
3038                                u16 port_per_thread, u32 snat_thread_index)
3039 {
3040   snat_main_t *sm = &snat_main;
3041   snat_address_t *a = addresses;
3042   u16 portnum, ports;
3043
3044   ports = sm->end_port - sm->start_port + 1;
3045
3046   if (!vec_len (addresses))
3047     goto exhausted;
3048
3049   switch (k->protocol)
3050     {
3051 #define _(N, i, n, s) \
3052     case SNAT_PROTOCOL_##N: \
3053       if (a->busy_##n##_ports < ports) \
3054         { \
3055           while (1) \
3056             { \
3057               portnum = snat_random_port(sm->start_port, sm->end_port); \
3058               if (a->busy_##n##_port_refcounts[portnum]) \
3059                 continue; \
3060               ++a->busy_##n##_port_refcounts[portnum]; \
3061               a->busy_##n##_ports++; \
3062               k->addr = a->addr; \
3063               k->port = clib_host_to_net_u16 (portnum); \
3064               return 0; \
3065             } \
3066         } \
3067       break;
3068       foreach_snat_protocol
3069 #undef _
3070     default:
3071       nat_elog_info ("unknown protocol");
3072       return 1;
3073     }
3074
3075 exhausted:
3076   /* Totally out of translations to use... */
3077   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3078   return 1;
3079 }
3080
3081 void
3082 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3083 {
3084   dpo_id_t dpo_v4 = DPO_INVALID;
3085   fib_prefix_t pfx = {
3086     .fp_proto = FIB_PROTOCOL_IP4,
3087     .fp_len = 32,
3088     .fp_addr.ip4.as_u32 = addr.as_u32,
3089   };
3090
3091   if (is_add)
3092     {
3093       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3094       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3095                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3096       dpo_reset (&dpo_v4);
3097     }
3098   else
3099     {
3100       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3101     }
3102 }
3103
3104 u8 *
3105 format_session_kvp (u8 * s, va_list * args)
3106 {
3107   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3108   snat_session_key_t k;
3109
3110   k.as_u64 = v->key;
3111
3112   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
3113
3114   return s;
3115 }
3116
3117 u8 *
3118 format_static_mapping_kvp (u8 * s, va_list * args)
3119 {
3120   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3121   snat_session_key_t k;
3122
3123   k.as_u64 = v->key;
3124
3125   s = format (s, "%U static-mapping-index %llu",
3126               format_static_mapping_key, &k, v->value);
3127
3128   return s;
3129 }
3130
3131 u8 *
3132 format_user_kvp (u8 * s, va_list * args)
3133 {
3134   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3135   snat_user_key_t k;
3136
3137   k.as_u64 = v->key;
3138
3139   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3140               k.fib_index, v->value);
3141
3142   return s;
3143 }
3144
3145 u8 *
3146 format_ed_session_kvp (u8 * s, va_list * args)
3147 {
3148   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3149   nat_ed_ses_key_t k;
3150
3151   k.as_u64[0] = v->key[0];
3152   k.as_u64[1] = v->key[1];
3153
3154   s =
3155     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3156             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
3157             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
3158             format_ip_protocol, k.proto, k.fib_index, v->value);
3159
3160   return s;
3161 }
3162
3163 static u32
3164 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3165                            u8 is_output)
3166 {
3167   snat_main_t *sm = &snat_main;
3168   u32 next_worker_index = 0;
3169   u32 hash;
3170
3171   next_worker_index = sm->first_worker_index;
3172   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3173     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3174
3175   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3176     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3177   else
3178     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3179
3180   return next_worker_index;
3181 }
3182
3183 static u32
3184 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3185                            u32 rx_fib_index0, u8 is_output)
3186 {
3187   snat_main_t *sm = &snat_main;
3188   udp_header_t *udp;
3189   u16 port;
3190   snat_session_key_t m_key;
3191   clib_bihash_kv_8_8_t kv, value;
3192   snat_static_mapping_t *m;
3193   u32 proto;
3194   u32 next_worker_index = 0;
3195
3196   /* first try static mappings without port */
3197   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3198     {
3199       m_key.addr = ip0->dst_address;
3200       m_key.port = 0;
3201       m_key.protocol = 0;
3202       m_key.fib_index = rx_fib_index0;
3203       kv.key = m_key.as_u64;
3204       if (!clib_bihash_search_8_8
3205           (&sm->static_mapping_by_external, &kv, &value))
3206         {
3207           m = pool_elt_at_index (sm->static_mappings, value.value);
3208           return m->workers[0];
3209         }
3210     }
3211
3212   proto = ip_proto_to_snat_proto (ip0->protocol);
3213   udp = ip4_next_header (ip0);
3214   port = udp->dst_port;
3215
3216   /* unknown protocol */
3217   if (PREDICT_FALSE (proto == ~0))
3218     {
3219       /* use current thread */
3220       return vlib_get_thread_index ();
3221     }
3222
3223   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3224     {
3225       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3226       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3227       if (!icmp_type_is_error_message
3228           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3229         port = vnet_buffer (b)->ip.reass.l4_src_port;
3230       else
3231         {
3232           /* if error message, then it's not fragmented and we can access it */
3233           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3234           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3235           void *l4_header = ip4_next_header (inner_ip);
3236           switch (proto)
3237             {
3238             case SNAT_PROTOCOL_ICMP:
3239               icmp = (icmp46_header_t *) l4_header;
3240               echo = (icmp_echo_header_t *) (icmp + 1);
3241               port = echo->identifier;
3242               break;
3243             case SNAT_PROTOCOL_UDP:
3244             case SNAT_PROTOCOL_TCP:
3245               port = ((tcp_udp_header_t *) l4_header)->src_port;
3246               break;
3247             default:
3248               return vlib_get_thread_index ();
3249             }
3250         }
3251     }
3252
3253   /* try static mappings with port */
3254   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3255     {
3256       m_key.addr = ip0->dst_address;
3257       m_key.port = clib_net_to_host_u16 (port);
3258       m_key.protocol = proto;
3259       m_key.fib_index = rx_fib_index0;
3260       kv.key = m_key.as_u64;
3261       if (!clib_bihash_search_8_8
3262           (&sm->static_mapping_by_external, &kv, &value))
3263         {
3264           m = pool_elt_at_index (sm->static_mappings, value.value);
3265           return m->workers[0];
3266         }
3267     }
3268
3269   /* worker by outside port */
3270   next_worker_index = sm->first_worker_index;
3271   next_worker_index +=
3272     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3273   return next_worker_index;
3274 }
3275
3276 static u32
3277 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3278                                u8 is_output)
3279 {
3280   snat_main_t *sm = &snat_main;
3281   u32 next_worker_index = sm->first_worker_index;
3282   u32 hash;
3283
3284   clib_bihash_kv_16_8_t kv16, value16;
3285   snat_main_per_thread_data_t *tsm;
3286   udp_header_t *udp;
3287
3288   if (PREDICT_FALSE (is_output))
3289     {
3290       u32 fib_index = sm->outside_fib_index;
3291       nat_outside_fib_t *outside_fib;
3292       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3293       fib_prefix_t pfx = {
3294         .fp_proto = FIB_PROTOCOL_IP4,
3295         .fp_len = 32,
3296         .fp_addr = {
3297                     .ip4.as_u32 = ip->dst_address.as_u32,
3298                     }
3299         ,
3300       };
3301
3302       udp = ip4_next_header (ip);
3303
3304       switch (vec_len (sm->outside_fibs))
3305         {
3306         case 0:
3307           fib_index = sm->outside_fib_index;
3308           break;
3309         case 1:
3310           fib_index = sm->outside_fibs[0].fib_index;
3311           break;
3312         default:
3313             /* *INDENT-OFF* */
3314             vec_foreach (outside_fib, sm->outside_fibs)
3315               {
3316                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3317                 if (FIB_NODE_INDEX_INVALID != fei)
3318                   {
3319                     if (fib_entry_get_resolving_interface (fei) != ~0)
3320                       {
3321                         fib_index = outside_fib->fib_index;
3322                         break;
3323                       }
3324                   }
3325               }
3326             /* *INDENT-ON* */
3327           break;
3328         }
3329
3330       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3331                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3332
3333       /* *INDENT-OFF* */
3334       vec_foreach (tsm, sm->per_thread_data)
3335         {
3336           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3337                                                       &kv16, &value16)))
3338             {
3339               next_worker_index += tsm->thread_index;
3340
3341               nat_elog_debug_handoff (
3342                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3343                 next_worker_index, fib_index,
3344                 clib_net_to_host_u32 (ip->src_address.as_u32),
3345                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3346
3347               return next_worker_index;
3348             }
3349         }
3350       /* *INDENT-ON* */
3351     }
3352
3353   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3354     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3355
3356   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3357     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3358   else
3359     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3360
3361   if (PREDICT_TRUE (!is_output))
3362     {
3363       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3364                               next_worker_index, rx_fib_index,
3365                               clib_net_to_host_u32 (ip->src_address.as_u32),
3366                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3367     }
3368   else
3369     {
3370       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3371                               next_worker_index, rx_fib_index,
3372                               clib_net_to_host_u32 (ip->src_address.as_u32),
3373                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3374     }
3375
3376   return next_worker_index;
3377 }
3378
3379 static u32
3380 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3381                                u32 rx_fib_index, u8 is_output)
3382 {
3383   snat_main_t *sm = &snat_main;
3384   clib_bihash_kv_8_8_t kv, value;
3385   clib_bihash_kv_16_8_t kv16, value16;
3386   snat_main_per_thread_data_t *tsm;
3387
3388   u32 proto, next_worker_index = 0;
3389   udp_header_t *udp;
3390   u16 port;
3391   snat_static_mapping_t *m;
3392   u32 hash;
3393
3394   proto = ip_proto_to_snat_proto (ip->protocol);
3395
3396   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3397     {
3398       udp = ip4_next_header (ip);
3399
3400       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3401                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3402
3403       /* *INDENT-OFF* */
3404       vec_foreach (tsm, sm->per_thread_data)
3405         {
3406           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3407                                                       &kv16, &value16)))
3408             {
3409               next_worker_index = sm->first_worker_index + tsm->thread_index;
3410               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3411                           next_worker_index, rx_fib_index,
3412                           clib_net_to_host_u32 (ip->src_address.as_u32),
3413                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3414               return next_worker_index;
3415             }
3416           }
3417         /* *INDENT-ON* */
3418     }
3419   else if (proto == SNAT_PROTOCOL_ICMP)
3420     {
3421       nat_ed_ses_key_t key;
3422
3423       if (!get_icmp_o2i_ed_key (b, ip, &key))
3424         {
3425
3426           key.fib_index = rx_fib_index;
3427           kv16.key[0] = key.as_u64[0];
3428           kv16.key[1] = key.as_u64[1];
3429
3430           /* *INDENT-OFF* */
3431           vec_foreach (tsm, sm->per_thread_data)
3432             {
3433               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3434                                                           &kv16, &value16)))
3435                 {
3436                   next_worker_index = sm->first_worker_index +
3437                                       tsm->thread_index;
3438                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3439                               next_worker_index, rx_fib_index,
3440                               clib_net_to_host_u32 (ip->src_address.as_u32),
3441                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3442                   return next_worker_index;
3443                 }
3444             }
3445           /* *INDENT-ON* */
3446         }
3447     }
3448
3449   /* first try static mappings without port */
3450   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3451     {
3452       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3453       if (!clib_bihash_search_8_8
3454           (&sm->static_mapping_by_external, &kv, &value))
3455         {
3456           m = pool_elt_at_index (sm->static_mappings, value.value);
3457           next_worker_index = m->workers[0];
3458           goto done;
3459         }
3460     }
3461
3462   /* unknown protocol */
3463   if (PREDICT_FALSE (proto == ~0))
3464     {
3465       /* use current thread */
3466       next_worker_index = vlib_get_thread_index ();
3467       goto done;
3468     }
3469
3470   udp = ip4_next_header (ip);
3471   port = udp->dst_port;
3472
3473   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3474     {
3475       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3476       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3477       if (!icmp_type_is_error_message
3478           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3479         port = vnet_buffer (b)->ip.reass.l4_src_port;
3480       else
3481         {
3482           /* if error message, then it's not fragmented and we can access it */
3483           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3484           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3485           void *l4_header = ip4_next_header (inner_ip);
3486           switch (proto)
3487             {
3488             case SNAT_PROTOCOL_ICMP:
3489               icmp = (icmp46_header_t *) l4_header;
3490               echo = (icmp_echo_header_t *) (icmp + 1);
3491               port = echo->identifier;
3492               break;
3493             case SNAT_PROTOCOL_UDP:
3494             case SNAT_PROTOCOL_TCP:
3495               port = ((tcp_udp_header_t *) l4_header)->src_port;
3496               break;
3497             default:
3498               next_worker_index = vlib_get_thread_index ();
3499               goto done;
3500             }
3501         }
3502     }
3503
3504   /* try static mappings with port */
3505   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3506     {
3507       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3508                   clib_net_to_host_u16 (port));
3509       if (!clib_bihash_search_8_8
3510           (&sm->static_mapping_by_external, &kv, &value))
3511         {
3512           m = pool_elt_at_index (sm->static_mappings, value.value);
3513           if (!is_lb_static_mapping (m))
3514             {
3515               next_worker_index = m->workers[0];
3516               goto done;
3517             }
3518
3519           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3520             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3521
3522           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3523             next_worker_index =
3524               m->workers[hash & (_vec_len (m->workers) - 1)];
3525           else
3526             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3527           goto done;
3528         }
3529     }
3530
3531   /* worker by outside port */
3532   next_worker_index = sm->first_worker_index;
3533   next_worker_index +=
3534     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3535
3536 done:
3537   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3538                           clib_net_to_host_u32 (ip->src_address.as_u32),
3539                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3540   return next_worker_index;
3541 }
3542
3543 void
3544 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3545                 ip4_address_t * out_addr, u16 out_port,
3546                 ip4_address_t * eh_addr, u16 eh_port,
3547                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3548                 u32 fib_index, u16 flags, u32 thread_index)
3549 {
3550   snat_main_t *sm = &snat_main;
3551   snat_session_key_t key;
3552   snat_user_t *u;
3553   snat_session_t *s;
3554   clib_bihash_kv_8_8_t kv;
3555   f64 now = vlib_time_now (sm->vlib_main);
3556   nat_outside_fib_t *outside_fib;
3557   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3558   snat_main_per_thread_data_t *tsm;
3559   fib_prefix_t pfx = {
3560     .fp_proto = FIB_PROTOCOL_IP4,
3561     .fp_len = 32,
3562     .fp_addr = {
3563                 .ip4.as_u32 = eh_addr->as_u32,
3564                 },
3565   };
3566
3567   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3568
3569   key.addr.as_u32 = out_addr->as_u32;
3570   key.port = out_port;
3571   key.protocol = proto;
3572
3573   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3574     {
3575       if (nat_set_outside_address_and_port
3576           (sm->addresses, thread_index, &key))
3577         return;
3578     }
3579
3580   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3581   if (!u)
3582     return;
3583
3584   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3585   if (!s)
3586     return;
3587
3588   s->last_heard = now;
3589   s->flags = flags;
3590   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3591   s->ext_host_port = eh_port;
3592   user_session_increment (sm, u, snat_is_session_static (s));
3593   switch (vec_len (sm->outside_fibs))
3594     {
3595     case 0:
3596       key.fib_index = sm->outside_fib_index;
3597       break;
3598     case 1:
3599       key.fib_index = sm->outside_fibs[0].fib_index;
3600       break;
3601     default:
3602       /* *INDENT-OFF* */
3603       vec_foreach (outside_fib, sm->outside_fibs)
3604         {
3605           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3606           if (FIB_NODE_INDEX_INVALID != fei)
3607             {
3608               if (fib_entry_get_resolving_interface (fei) != ~0)
3609                 {
3610                   key.fib_index = outside_fib->fib_index;
3611                   break;
3612                 }
3613             }
3614         }
3615       /* *INDENT-ON* */
3616       break;
3617     }
3618   s->out2in = key;
3619   kv.key = key.as_u64;
3620   kv.value = s - tsm->sessions;
3621   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3622     nat_elog_warn ("out2in key add failed");
3623
3624   key.addr.as_u32 = in_addr->as_u32;
3625   key.port = in_port;
3626   key.fib_index = fib_index;
3627   s->in2out = key;
3628   kv.key = key.as_u64;
3629   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3630     nat_elog_warn ("in2out key add failed");
3631 }
3632
3633 void
3634 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3635                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3636                 u32 ti)
3637 {
3638   snat_main_t *sm = &snat_main;
3639   snat_session_key_t key;
3640   clib_bihash_kv_8_8_t kv, value;
3641   u32 thread_index;
3642   snat_session_t *s;
3643   snat_main_per_thread_data_t *tsm;
3644
3645   if (sm->num_workers > 1)
3646     thread_index =
3647       sm->first_worker_index +
3648       (sm->workers[(clib_net_to_host_u16 (out_port) -
3649                     1024) / sm->port_per_thread]);
3650   else
3651     thread_index = sm->num_workers;
3652   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3653
3654   key.addr.as_u32 = out_addr->as_u32;
3655   key.port = out_port;
3656   key.protocol = proto;
3657   key.fib_index = fib_index;
3658   kv.key = key.as_u64;
3659   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3660     return;
3661
3662   s = pool_elt_at_index (tsm->sessions, value.value);
3663   nat_free_session_data (sm, s, thread_index, 1);
3664   nat44_delete_session (sm, s, thread_index);
3665 }
3666
3667 void
3668 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3669                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3670                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3671 {
3672   snat_main_t *sm = &snat_main;
3673   snat_session_key_t key;
3674   clib_bihash_kv_8_8_t kv, value;
3675   snat_session_t *s;
3676   snat_main_per_thread_data_t *tsm;
3677
3678   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3679
3680   key.addr.as_u32 = out_addr->as_u32;
3681   key.port = out_port;
3682   key.protocol = proto;
3683   key.fib_index = fib_index;
3684   kv.key = key.as_u64;
3685   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3686     return;
3687
3688   s = pool_elt_at_index (tsm->sessions, value.value);
3689   s->total_pkts = total_pkts;
3690   s->total_bytes = total_bytes;
3691 }
3692
3693 void
3694 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3695                    ip4_address_t * out_addr, u16 out_port,
3696                    ip4_address_t * eh_addr, u16 eh_port,
3697                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3698                    u32 fib_index, u16 flags, u32 thread_index)
3699 {
3700   snat_main_t *sm = &snat_main;
3701   snat_session_key_t key;
3702   snat_user_t *u;
3703   snat_session_t *s;
3704   clib_bihash_kv_16_8_t kv;
3705   f64 now = vlib_time_now (sm->vlib_main);
3706   nat_outside_fib_t *outside_fib;
3707   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3708   snat_main_per_thread_data_t *tsm;
3709   fib_prefix_t pfx = {
3710     .fp_proto = FIB_PROTOCOL_IP4,
3711     .fp_len = 32,
3712     .fp_addr = {
3713                 .ip4.as_u32 = eh_addr->as_u32,
3714                 },
3715   };
3716
3717   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3718
3719   key.addr.as_u32 = out_addr->as_u32;
3720   key.port = out_port;
3721   key.protocol = proto;
3722
3723   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3724     {
3725       if (nat_set_outside_address_and_port
3726           (sm->addresses, thread_index, &key))
3727         return;
3728     }
3729
3730   key.addr.as_u32 = ehn_addr->as_u32;
3731   key.port = ehn_port;
3732   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3733     {
3734       if (nat_set_outside_address_and_port
3735           (sm->twice_nat_addresses, thread_index, &key))
3736         return;
3737     }
3738
3739   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3740   if (!u)
3741     return;
3742
3743   s = nat_ed_session_alloc (sm, u, thread_index, now);
3744   if (!s)
3745     return;
3746
3747   s->last_heard = now;
3748   s->flags = flags;
3749   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3750   s->ext_host_nat_port = s->ext_host_port = eh_port;
3751   if (is_twice_nat_session (s))
3752     {
3753       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3754       s->ext_host_nat_port = ehn_port;
3755     }
3756   user_session_increment (sm, u, snat_is_session_static (s));
3757   switch (vec_len (sm->outside_fibs))
3758     {
3759     case 0:
3760       key.fib_index = sm->outside_fib_index;
3761       break;
3762     case 1:
3763       key.fib_index = sm->outside_fibs[0].fib_index;
3764       break;
3765     default:
3766       /* *INDENT-OFF* */
3767       vec_foreach (outside_fib, sm->outside_fibs)
3768         {
3769           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3770           if (FIB_NODE_INDEX_INVALID != fei)
3771             {
3772               if (fib_entry_get_resolving_interface (fei) != ~0)
3773                 {
3774                   key.fib_index = outside_fib->fib_index;
3775                   break;
3776                 }
3777             }
3778         }
3779       /* *INDENT-ON* */
3780       break;
3781     }
3782   key.addr.as_u32 = out_addr->as_u32;
3783   key.port = out_port;
3784   s->out2in = key;
3785   kv.value = s - tsm->sessions;
3786
3787   key.addr.as_u32 = in_addr->as_u32;
3788   key.port = in_port;
3789   key.fib_index = fib_index;
3790   s->in2out = key;
3791
3792   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3793               snat_proto_to_ip_proto (proto), fib_index, in_port,
3794               s->ext_host_nat_port);
3795   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3796     nat_elog_warn ("in2out key add failed");
3797
3798   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3799               s->out2in.fib_index, out_port, eh_port);
3800   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3801     nat_elog_warn ("out2in key add failed");
3802 }
3803
3804 void
3805 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3806                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3807                    u32 fib_index, u32 ti)
3808 {
3809   snat_main_t *sm = &snat_main;
3810   nat_ed_ses_key_t key;
3811   clib_bihash_kv_16_8_t kv, value;
3812   u32 thread_index;
3813   snat_session_t *s;
3814   snat_main_per_thread_data_t *tsm;
3815
3816   if (sm->num_workers > 1)
3817     thread_index =
3818       sm->first_worker_index +
3819       (sm->workers[(clib_net_to_host_u16 (out_port) -
3820                     1024) / sm->port_per_thread]);
3821   else
3822     thread_index = sm->num_workers;
3823   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3824
3825   key.l_addr.as_u32 = out_addr->as_u32;
3826   key.l_port = out_port;
3827   key.r_addr.as_u32 = eh_addr->as_u32;
3828   key.r_port = eh_port;
3829   key.proto = proto;
3830   key.fib_index = fib_index;
3831   kv.key[0] = key.as_u64[0];
3832   kv.key[1] = key.as_u64[1];
3833   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3834     return;
3835
3836   s = pool_elt_at_index (tsm->sessions, value.value);
3837   nat_free_session_data (sm, s, thread_index, 1);
3838   nat44_delete_session (sm, s, thread_index);
3839 }
3840
3841 void
3842 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3843                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3844                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3845                    u32 thread_index)
3846 {
3847   snat_main_t *sm = &snat_main;
3848   nat_ed_ses_key_t key;
3849   clib_bihash_kv_16_8_t kv, value;
3850   snat_session_t *s;
3851   snat_main_per_thread_data_t *tsm;
3852
3853   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3854
3855   key.l_addr.as_u32 = out_addr->as_u32;
3856   key.l_port = out_port;
3857   key.r_addr.as_u32 = eh_addr->as_u32;
3858   key.r_port = eh_port;
3859   key.proto = proto;
3860   key.fib_index = fib_index;
3861   kv.key[0] = key.as_u64[0];
3862   kv.key[1] = key.as_u64[1];
3863   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3864     return;
3865
3866   s = pool_elt_at_index (tsm->sessions, value.value);
3867   s->total_pkts = total_pkts;
3868   s->total_bytes = total_bytes;
3869 }
3870
3871 static clib_error_t *
3872 snat_config (vlib_main_t * vm, unformat_input_t * input)
3873 {
3874   snat_main_t *sm = &snat_main;
3875   nat66_main_t *nm = &nat66_main;
3876   //dslite_main_t *dm = &dslite_main;
3877   snat_main_per_thread_data_t *tsm;
3878
3879   u32 static_mapping_buckets = 1024;
3880   u32 static_mapping_memory_size = 64 << 20;
3881
3882   u32 nat64_bib_buckets = 1024;
3883   u32 nat64_bib_memory_size = 128 << 20;
3884
3885   u32 nat64_st_buckets = 2048;
3886   u32 nat64_st_memory_size = 256 << 20;
3887
3888   u32 user_buckets = 128;
3889   u32 user_memory_size = 64 << 20;
3890   u32 translation_buckets = 1024;
3891   u32 translation_memory_size = 128 << 20;
3892
3893   u32 max_translations_per_user = ~0;
3894
3895   u32 outside_vrf_id = 0;
3896   u32 outside_ip6_vrf_id = 0;
3897   u32 inside_vrf_id = 0;
3898   u8 static_mapping_only = 0;
3899   u8 static_mapping_connection_tracking = 0;
3900
3901   // configurable timeouts
3902   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3903   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3904   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3905   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3906
3907   sm->deterministic = 0;
3908   sm->out2in_dpo = 0;
3909   sm->endpoint_dependent = 0;
3910
3911   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3912     {
3913       if (unformat
3914           (input, "translation hash buckets %d", &translation_buckets))
3915         ;
3916       else if (unformat (input, "udp timeout %d", &udp_timeout))
3917         ;
3918       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3919         ;
3920       else if (unformat (input, "tcp transitory timeout %d",
3921                          &tcp_transitory_timeout));
3922       else if (unformat (input, "tcp established timeout %d",
3923                          &tcp_established_timeout));
3924       else if (unformat (input, "translation hash memory %d",
3925                          &translation_memory_size));
3926       else if (unformat (input, "user hash buckets %d", &user_buckets))
3927         ;
3928       else if (unformat (input, "user hash memory %d", &user_memory_size))
3929         ;
3930       else if (unformat (input, "max translations per user %d",
3931                          &max_translations_per_user))
3932         ;
3933       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3934         ;
3935       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3936         ;
3937       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3938         ;
3939       else if (unformat (input, "static mapping only"))
3940         {
3941           static_mapping_only = 1;
3942           if (unformat (input, "connection tracking"))
3943             static_mapping_connection_tracking = 1;
3944         }
3945       else if (unformat (input, "deterministic"))
3946         sm->deterministic = 1;
3947       else if (unformat (input, "nat64 bib hash buckets %d",
3948                          &nat64_bib_buckets))
3949         ;
3950       else if (unformat (input, "nat64 bib hash memory %d",
3951                          &nat64_bib_memory_size))
3952         ;
3953       else
3954         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3955         ;
3956       else if (unformat (input, "nat64 st hash memory %d",
3957                          &nat64_st_memory_size))
3958         ;
3959       else if (unformat (input, "out2in dpo"))
3960         sm->out2in_dpo = 1;
3961       //else if (unformat (input, "dslite ce"))
3962       //dslite_set_ce (dm, 1);
3963       else if (unformat (input, "endpoint-dependent"))
3964         sm->endpoint_dependent = 1;
3965       else
3966         return clib_error_return (0, "unknown input '%U'",
3967                                   format_unformat_error, input);
3968     }
3969
3970   if (sm->deterministic && sm->endpoint_dependent)
3971     return clib_error_return (0,
3972                               "deterministic and endpoint-dependent modes are mutually exclusive");
3973
3974   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3975     return clib_error_return (0,
3976                               "static mapping only mode available only for simple nat");
3977
3978   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3979     return clib_error_return (0,
3980                               "out2in dpo mode available only for simple nat");
3981
3982   /* optionally configurable timeouts for testing purposes */
3983   sm->udp_timeout = udp_timeout;
3984   sm->tcp_transitory_timeout = tcp_transitory_timeout;
3985   sm->tcp_established_timeout = tcp_established_timeout;
3986   sm->icmp_timeout = icmp_timeout;
3987
3988   sm->min_timeout = nat44_minimal_timeout (sm);
3989
3990   sm->user_buckets = user_buckets;
3991   sm->user_memory_size = user_memory_size;
3992
3993   sm->translation_buckets = translation_buckets;
3994   sm->translation_memory_size = translation_memory_size;
3995
3996   /* do not exceed load factor 10 */
3997   sm->max_translations = 10 * translation_buckets;
3998   sm->max_translations_per_user = max_translations_per_user == ~0 ?
3999     sm->max_translations : max_translations_per_user;
4000
4001   sm->outside_vrf_id = outside_vrf_id;
4002   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4003                                                              outside_vrf_id,
4004                                                              nat_fib_src_hi);
4005   nm->outside_vrf_id = outside_ip6_vrf_id;
4006   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
4007                                                              outside_ip6_vrf_id,
4008                                                              nat_fib_src_hi);
4009   sm->inside_vrf_id = inside_vrf_id;
4010   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4011                                                             inside_vrf_id,
4012                                                             nat_fib_src_hi);
4013   sm->static_mapping_only = static_mapping_only;
4014   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4015
4016   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4017                   nat64_st_memory_size);
4018
4019   if (sm->deterministic)
4020     {
4021       sm->in2out_node_index = snat_det_in2out_node.index;
4022       sm->in2out_output_node_index = ~0;
4023       sm->out2in_node_index = snat_det_out2in_node.index;
4024       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4025       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4026     }
4027   else
4028     {
4029       if (sm->endpoint_dependent)
4030         {
4031           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4032           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4033
4034           sm->handoff_out2in_index = nat_pre_out2in_node.index;
4035           sm->handoff_in2out_index = nat_pre_in2out_node.index;
4036           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
4037
4038           sm->in2out_node_index = nat44_ed_in2out_node.index;
4039           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4040           sm->out2in_node_index = nat44_ed_out2in_node.index;
4041
4042           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4043           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4044           nat_affinity_init (vm);
4045           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4046                        nat_ha_sref_ed_cb);
4047         }
4048       else
4049         {
4050           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4051           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4052
4053           sm->handoff_out2in_index = snat_out2in_node.index;
4054           sm->handoff_in2out_index = snat_in2out_node.index;
4055           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
4056
4057           sm->in2out_node_index = snat_in2out_node.index;
4058           sm->in2out_output_node_index = snat_in2out_output_node.index;
4059           sm->out2in_node_index = snat_out2in_node.index;
4060           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4061           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4062           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4063         }
4064       if (!static_mapping_only ||
4065           (static_mapping_only && static_mapping_connection_tracking))
4066         {
4067           /* *INDENT-OFF* */
4068           vec_foreach (tsm, sm->per_thread_data)
4069             {
4070               tsm->min_session_timeout = 0;
4071
4072               tsm->cleared = 0;
4073               tsm->cleanup_runs = 0;
4074               tsm->cleanup_timeout = 0;
4075
4076               if (sm->endpoint_dependent)
4077                 {
4078                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
4079                                          translation_buckets,
4080                                          translation_memory_size);
4081                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
4082                                                       format_ed_session_kvp);
4083
4084                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
4085                                          translation_buckets,
4086                                          translation_memory_size);
4087                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
4088                                                       format_ed_session_kvp);
4089                   clib_bihash_init_16_8
4090                   (&sm->ed_ext_ports, "ed-nat-5-tuple-port-overload-hash",
4091                    translation_buckets, translation_memory_size);
4092                 }
4093               else
4094                 {
4095                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
4096                                         translation_buckets,
4097                                         translation_memory_size);
4098                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
4099                                                      format_session_kvp);
4100
4101                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
4102                                         translation_buckets,
4103                                         translation_memory_size);
4104                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
4105                                                      format_session_kvp);
4106                 }
4107
4108               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
4109                                     user_memory_size);
4110               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
4111                                                  format_user_kvp);
4112             }
4113           /* *INDENT-ON* */
4114
4115         }
4116       else
4117         {
4118           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4119           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4120         }
4121       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4122                             "static_mapping_by_local", static_mapping_buckets,
4123                             static_mapping_memory_size);
4124       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4125                                          format_static_mapping_kvp);
4126
4127       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4128                             "static_mapping_by_external",
4129                             static_mapping_buckets,
4130                             static_mapping_memory_size);
4131       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4132                                          format_static_mapping_kvp);
4133     }
4134
4135   return 0;
4136 }
4137
4138 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4139
4140 static void
4141 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4142                                  uword opaque,
4143                                  u32 sw_if_index,
4144                                  ip4_address_t * address,
4145                                  u32 address_length,
4146                                  u32 if_address_index, u32 is_delete)
4147 {
4148   snat_main_t *sm = &snat_main;
4149   snat_static_map_resolve_t *rp;
4150   snat_static_mapping_t *m;
4151   snat_session_key_t m_key;
4152   clib_bihash_kv_8_8_t kv, value;
4153   int i, rv;
4154   ip4_address_t l_addr;
4155
4156   for (i = 0; i < vec_len (sm->to_resolve); i++)
4157     {
4158       rp = sm->to_resolve + i;
4159       if (rp->addr_only == 0)
4160         continue;
4161       if (rp->sw_if_index == sw_if_index)
4162         goto match;
4163     }
4164
4165   return;
4166
4167 match:
4168   m_key.addr.as_u32 = address->as_u32;
4169   m_key.port = rp->addr_only ? 0 : rp->e_port;
4170   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4171   m_key.fib_index = sm->outside_fib_index;
4172   kv.key = m_key.as_u64;
4173   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4174     m = 0;
4175   else
4176     m = pool_elt_at_index (sm->static_mappings, value.value);
4177
4178   if (!is_delete)
4179     {
4180       /* Don't trip over lease renewal, static config */
4181       if (m)
4182         return;
4183     }
4184   else
4185     {
4186       if (!m)
4187         return;
4188     }
4189
4190   /* Indetity mapping? */
4191   if (rp->l_addr.as_u32 == 0)
4192     l_addr.as_u32 = address[0].as_u32;
4193   else
4194     l_addr.as_u32 = rp->l_addr.as_u32;
4195   /* Add the static mapping */
4196   rv = snat_add_static_mapping (l_addr,
4197                                 address[0],
4198                                 rp->l_port,
4199                                 rp->e_port,
4200                                 rp->vrf_id,
4201                                 rp->addr_only, ~0 /* sw_if_index */ ,
4202                                 rp->proto, !is_delete, rp->twice_nat,
4203                                 rp->out2in_only, rp->tag, rp->identity_nat);
4204   if (rv)
4205     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4206 }
4207
4208 static void
4209 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4210                                        uword opaque,
4211                                        u32 sw_if_index,
4212                                        ip4_address_t * address,
4213                                        u32 address_length,
4214                                        u32 if_address_index, u32 is_delete)
4215 {
4216   snat_main_t *sm = &snat_main;
4217   snat_static_map_resolve_t *rp;
4218   ip4_address_t l_addr;
4219   int i, j;
4220   int rv;
4221   u8 twice_nat = 0;
4222   snat_address_t *addresses = sm->addresses;
4223
4224   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4225     {
4226       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4227         goto match;
4228     }
4229
4230   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4231     {
4232       twice_nat = 1;
4233       addresses = sm->twice_nat_addresses;
4234       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4235         goto match;
4236     }
4237
4238   return;
4239
4240 match:
4241   if (!is_delete)
4242     {
4243       /* Don't trip over lease renewal, static config */
4244       for (j = 0; j < vec_len (addresses); j++)
4245         if (addresses[j].addr.as_u32 == address->as_u32)
4246           return;
4247
4248       (void) snat_add_address (sm, address, ~0, twice_nat);
4249       /* Scan static map resolution vector */
4250       for (j = 0; j < vec_len (sm->to_resolve); j++)
4251         {
4252           rp = sm->to_resolve + j;
4253           if (rp->addr_only)
4254             continue;
4255           /* On this interface? */
4256           if (rp->sw_if_index == sw_if_index)
4257             {
4258               /* Indetity mapping? */
4259               if (rp->l_addr.as_u32 == 0)
4260                 l_addr.as_u32 = address[0].as_u32;
4261               else
4262                 l_addr.as_u32 = rp->l_addr.as_u32;
4263               /* Add the static mapping */
4264               rv = snat_add_static_mapping (l_addr,
4265                                             address[0],
4266                                             rp->l_port,
4267                                             rp->e_port,
4268                                             rp->vrf_id,
4269                                             rp->addr_only,
4270                                             ~0 /* sw_if_index */ ,
4271                                             rp->proto,
4272                                             rp->is_add, rp->twice_nat,
4273                                             rp->out2in_only, rp->tag,
4274                                             rp->identity_nat);
4275               if (rv)
4276                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4277                                     "i4", rv);
4278             }
4279         }
4280       return;
4281     }
4282   else
4283     {
4284       (void) snat_del_address (sm, address[0], 1, twice_nat);
4285       return;
4286     }
4287 }
4288
4289
4290 int
4291 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4292                             u8 twice_nat)
4293 {
4294   ip4_main_t *ip4_main = sm->ip4_main;
4295   ip4_address_t *first_int_addr;
4296   snat_static_map_resolve_t *rp;
4297   u32 *indices_to_delete = 0;
4298   int i, j;
4299   u32 *auto_add_sw_if_indices =
4300     twice_nat ? sm->
4301     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4302
4303   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4304     );
4305
4306   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4307     {
4308       if (auto_add_sw_if_indices[i] == sw_if_index)
4309         {
4310           if (is_del)
4311             {
4312               /* if have address remove it */
4313               if (first_int_addr)
4314                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4315               else
4316                 {
4317                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4318                     {
4319                       rp = sm->to_resolve + j;
4320                       if (rp->sw_if_index == sw_if_index)
4321                         vec_add1 (indices_to_delete, j);
4322                     }
4323                   if (vec_len (indices_to_delete))
4324                     {
4325                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4326                         vec_del1 (sm->to_resolve, j);
4327                       vec_free (indices_to_delete);
4328                     }
4329                 }
4330               if (twice_nat)
4331                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4332               else
4333                 vec_del1 (sm->auto_add_sw_if_indices, i);
4334             }
4335           else
4336             return VNET_API_ERROR_VALUE_EXIST;
4337
4338           return 0;
4339         }
4340     }
4341
4342   if (is_del)
4343     return VNET_API_ERROR_NO_SUCH_ENTRY;
4344
4345   /* add to the auto-address list */
4346   if (twice_nat)
4347     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4348   else
4349     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4350
4351   /* If the address is already bound - or static - add it now */
4352   if (first_int_addr)
4353     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4354
4355   return 0;
4356 }
4357
4358 int
4359 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4360                    snat_protocol_t proto, u32 vrf_id, int is_in)
4361 {
4362   snat_main_per_thread_data_t *tsm;
4363   clib_bihash_kv_8_8_t kv, value;
4364   ip4_header_t ip;
4365   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4366   snat_session_key_t key;
4367   snat_session_t *s;
4368   clib_bihash_8_8_t *t;
4369
4370   if (sm->endpoint_dependent)
4371     return VNET_API_ERROR_UNSUPPORTED;
4372
4373   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4374   if (sm->num_workers > 1)
4375     tsm =
4376       vec_elt_at_index (sm->per_thread_data,
4377                         sm->worker_in2out_cb (&ip, fib_index, 0));
4378   else
4379     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4380
4381   key.addr.as_u32 = addr->as_u32;
4382   key.port = clib_host_to_net_u16 (port);
4383   key.protocol = proto;
4384   key.fib_index = fib_index;
4385   kv.key = key.as_u64;
4386   t = is_in ? &tsm->in2out : &tsm->out2in;
4387   if (!clib_bihash_search_8_8 (t, &kv, &value))
4388     {
4389       if (pool_is_free_index (tsm->sessions, value.value))
4390         return VNET_API_ERROR_UNSPECIFIED;
4391
4392       s = pool_elt_at_index (tsm->sessions, value.value);
4393       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4394       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4395       return 0;
4396     }
4397
4398   return VNET_API_ERROR_NO_SUCH_ENTRY;
4399 }
4400
4401 int
4402 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4403                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4404                       u32 vrf_id, int is_in)
4405 {
4406   ip4_header_t ip;
4407   clib_bihash_16_8_t *t;
4408   nat_ed_ses_key_t key;
4409   clib_bihash_kv_16_8_t kv, value;
4410   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4411   snat_session_t *s;
4412   snat_main_per_thread_data_t *tsm;
4413
4414   if (!sm->endpoint_dependent)
4415     return VNET_API_ERROR_FEATURE_DISABLED;
4416
4417   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4418   if (sm->num_workers > 1)
4419     tsm =
4420       vec_elt_at_index (sm->per_thread_data,
4421                         sm->worker_in2out_cb (&ip, fib_index, 0));
4422   else
4423     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4424
4425   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4426   key.l_addr.as_u32 = addr->as_u32;
4427   key.r_addr.as_u32 = eh_addr->as_u32;
4428   key.l_port = clib_host_to_net_u16 (port);
4429   key.r_port = clib_host_to_net_u16 (eh_port);
4430   key.proto = proto;
4431   key.fib_index = fib_index;
4432   kv.key[0] = key.as_u64[0];
4433   kv.key[1] = key.as_u64[1];
4434   if (clib_bihash_search_16_8 (t, &kv, &value))
4435     return VNET_API_ERROR_NO_SUCH_ENTRY;
4436
4437   if (pool_is_free_index (tsm->sessions, value.value))
4438     return VNET_API_ERROR_UNSPECIFIED;
4439   s = pool_elt_at_index (tsm->sessions, value.value);
4440   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4441   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4442   return 0;
4443 }
4444
4445 void
4446 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4447 {
4448   snat_main_t *sm = &snat_main;
4449
4450   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4451   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4452   sm->psid = psid;
4453   sm->psid_offset = psid_offset;
4454   sm->psid_length = psid_length;
4455 }
4456
4457 void
4458 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4459 {
4460   snat_main_t *sm = &snat_main;
4461
4462   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4463   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4464   sm->start_port = start_port;
4465   sm->end_port = end_port;
4466 }
4467
4468 void
4469 nat_set_alloc_addr_and_port_default (void)
4470 {
4471   snat_main_t *sm = &snat_main;
4472
4473   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4474   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4475 }
4476
4477 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4478                                  vlib_node_runtime_t * node,
4479                                  vlib_frame_t * frame)
4480 {
4481   return 0;
4482 }
4483
4484 /* *INDENT-OFF* */
4485 VLIB_REGISTER_NODE (nat_default_node) = {
4486   .name = "nat-default",
4487   .vector_size = sizeof (u32),
4488   .format_trace = 0,
4489   .type = VLIB_NODE_TYPE_INTERNAL,
4490   .n_errors = 0,
4491   .n_next_nodes = NAT_N_NEXT,
4492   .next_nodes = {
4493     [NAT_NEXT_DROP] = "error-drop",
4494     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4495     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4496     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4497     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4498     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4499     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4500     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4501     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4502   },
4503 };
4504 /* *INDENT-ON* */
4505
4506 /*
4507  * fd.io coding-style-patch-verification: ON
4508  *
4509  * Local Variables:
4510  * eval: (c-set-style "gnu")
4511  * End:
4512  */