src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36 #include <vppinfra/bihash_16_8.h>
  37 #include <nat/nat44/ed_inlines.h>
  38
  39 #include <vpp/app/version.h>
  40
  41 snat_main_t snat_main;
  42
  43 fib_source_t nat_fib_src_hi;
  44 fib_source_t nat_fib_src_low;
  45
  46 /* *INDENT-OFF* */
  47 /* Hook up input features */
  48 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat-pre-in2out",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-sv-reassembly-feature"),
  53 };
  54 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat-pre-out2in",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  58                                "ip4-dhcp-client-detect",
  59                                "ip4-sv-reassembly-feature"),
  60 };
  61 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  62   .arc_name = "ip4-unicast",
  63   .node_name = "nat44-in2out-worker-handoff",
  64   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  65 };
  66 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  67   .arc_name = "ip4-unicast",
  68   .node_name = "nat44-out2in-worker-handoff",
  69   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  70                                "ip4-dhcp-client-detect"),
  71 };
  72 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  73   .arc_name = "ip4-unicast",
  74   .node_name = "nat44-in2out",
  75   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  76 };
  77 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  78   .arc_name = "ip4-unicast",
  79   .node_name = "nat44-out2in",
  80   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  81                                "ip4-dhcp-client-detect"),
  82 };
  83 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-det-in2out",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  92 };
  93 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  94   .arc_name = "ip4-unicast",
  95   .node_name = "nat44-det-out2in",
  96   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  97                                "ip4-dhcp-client-detect"),
  98 };
  99 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
 100   .arc_name = "ip4-unicast",
 101   .node_name = "nat44-det-classify",
 102   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-ed-in2out",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 108 };
 109 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-ed-out2in",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-classify",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-handoff-classify",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 124 };
 125 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 126   .arc_name = "ip4-unicast",
 127   .node_name = "nat44-in2out-fast",
 128   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 129 };
 130 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 131   .arc_name = "ip4-unicast",
 132   .node_name = "nat44-out2in-fast",
 133   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 134                                "ip4-dhcp-client-detect"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 137   .arc_name = "ip4-unicast",
 138   .node_name = "nat44-hairpin-dst",
 139   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 140 };
 141 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 142   .arc_name = "ip4-unicast",
 143   .node_name = "nat44-ed-hairpin-dst",
 144   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 145 };
 146
 147 /* Hook up output features */
 148 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-in2out-output",
 151   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 152 };
 153 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 154   .arc_name = "ip4-output",
 155   .node_name = "nat44-in2out-output-worker-handoff",
 156   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 157 };
 158 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 159   .arc_name = "ip4-output",
 160   .node_name = "nat44-hairpin-src",
 161   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 164   .arc_name = "ip4-output",
 165   .node_name = "nat44-ed-in2out-output",
 166   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 167   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 168 };
 169 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 170   .arc_name = "ip4-output",
 171   .node_name = "nat44-ed-hairpin-src",
 172   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 173   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 174 };
 175
 176 /* Hook up ip4-local features */
 177 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 178 {
 179   .arc_name = "ip4-local",
 180   .node_name = "nat44-hairpinning",
 181   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 182 };
 183 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 184 {
 185   .arc_name = "ip4-local",
 186   .node_name = "nat44-ed-hairpinning",
 187   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 188 };
 189
 190
 191 VLIB_PLUGIN_REGISTER () = {
 192     .version = VPP_BUILD_VER,
 193     .description = "Network Address Translation (NAT)",
 194 };
 195 /* *INDENT-ON* */
 196
 197 void
 198 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 199                        u8 is_ha)
 200 {
 201   clib_bihash_kv_8_8_t kv;
 202   u8 proto;
 203   u16 r_port, l_port;
 204   ip4_address_t *l_addr, *r_addr;
 205   u32 fib_index = 0;
 206   clib_bihash_kv_16_8_t ed_kv;
 207   snat_main_per_thread_data_t *tsm =
 208     vec_elt_at_index (sm->per_thread_data, thread_index);
 209
 210   if (is_fwd_bypass_session (s))
 211     {
 212       if (snat_is_unk_proto_session (s))
 213         {
 214           init_ed_k (&ed_kv, s->in2out.addr, 0, s->ext_host_addr, 0, 0,
 215                      s->in2out.port);
 216         }
 217       else
 218         {
 219           l_port = s->in2out.port;
 220           r_port = s->ext_host_port;
 221           l_addr = &s->in2out.addr;
 222           r_addr = &s->ext_host_addr;
 223           proto = nat_proto_to_ip_proto (s->nat_proto);
 224           init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index,
 225                      proto);
 226         }
 227       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 228         nat_elog_warn ("in2out_ed key del failed");
 229       return;
 230     }
 231
 232   /* session lookup tables */
 233   if (is_ed_session (s))
 234     {
 235       if (is_affinity_sessions (s))
 236         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 237                              s->nat_proto, s->out2in.port);
 238       l_addr = &s->out2in.addr;
 239       r_addr = &s->ext_host_addr;
 240       fib_index = s->out2in.fib_index;
 241       if (snat_is_unk_proto_session (s))
 242         {
 243           proto = s->in2out.port;
 244           r_port = 0;
 245           l_port = 0;
 246         }
 247       else
 248         {
 249           proto = nat_proto_to_ip_proto (s->nat_proto);
 250           l_port = s->out2in.port;
 251           r_port = s->ext_host_port;
 252         }
 253       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 254       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 255         nat_elog_warn ("out2in_ed key del failed");
 256       l_addr = &s->in2out.addr;
 257       fib_index = s->in2out.fib_index;
 258       if (!snat_is_unk_proto_session (s))
 259         l_port = s->in2out.port;
 260       if (is_twice_nat_session (s))
 261         {
 262           r_addr = &s->ext_host_nat_addr;
 263           r_port = s->ext_host_nat_port;
 264         }
 265       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 266       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 267         nat_elog_warn ("in2out_ed key del failed");
 268
 269       if (!is_ha)
 270         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 271                                &s->in2out.addr, s->in2out.port,
 272                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 273                                &s->out2in.addr, s->out2in.port,
 274                                &s->ext_host_addr, s->ext_host_port,
 275                                s->nat_proto, is_twice_nat_session (s));
 276     }
 277   else
 278     {
 279       init_nat_i2o_k (&kv, s);
 280       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 281         nat_elog_warn ("in2out key del failed");
 282       init_nat_o2i_k (&kv, s);
 283       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 284         nat_elog_warn ("out2in key del failed");
 285
 286       if (!is_ha)
 287         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 288                                  &s->in2out.addr, s->in2out.port,
 289                                  &s->out2in.addr, s->out2in.port,
 290                                  s->nat_proto);
 291     }
 292
 293   if (snat_is_unk_proto_session (s))
 294     return;
 295
 296   if (!is_ha)
 297     {
 298       /* log NAT event */
 299       snat_ipfix_logging_nat44_ses_delete (thread_index,
 300                                            s->in2out.addr.as_u32,
 301                                            s->out2in.addr.as_u32,
 302                                            s->nat_proto,
 303                                            s->in2out.port,
 304                                            s->out2in.port,
 305                                            s->in2out.fib_index);
 306
 307       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 308                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 309                    thread_index);
 310     }
 311
 312   /* Twice NAT address and port for external host */
 313   if (is_twice_nat_session (s))
 314     {
 315       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 316                                           thread_index,
 317                                           &s->ext_host_nat_addr,
 318                                           s->ext_host_nat_port, s->nat_proto);
 319     }
 320
 321   if (snat_is_session_static (s))
 322     return;
 323
 324   snat_free_outside_address_and_port (sm->addresses, thread_index,
 325                                       &s->out2in.addr, s->out2in.port,
 326                                       s->nat_proto);
 327 }
 328
 329 int
 330 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
 331 {
 332   snat_main_t *sm = &snat_main;
 333   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
 334   u32 len = vec_len (sm->max_translations_per_fib);
 335
 336   if (len <= fib_index)
 337     {
 338       vec_validate (sm->max_translations_per_fib, fib_index + 1);
 339
 340       for (; len < vec_len (sm->max_translations_per_fib); len++)
 341         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
 342     }
 343
 344   sm->max_translations_per_fib[fib_index] = session_limit;
 345   return 0;
 346 }
 347
 348 void
 349 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 350                          u32 thread_index, u8 is_ha)
 351 {
 352   u8 proto;
 353   u16 r_port, l_port;
 354   ip4_address_t *l_addr, *r_addr;
 355   u32 fib_index;
 356   clib_bihash_kv_16_8_t ed_kv;
 357   snat_main_per_thread_data_t *tsm =
 358     vec_elt_at_index (sm->per_thread_data, thread_index);
 359
 360   if (is_fwd_bypass_session (s))
 361     {
 362       if (snat_is_unk_proto_session (s))
 363         {
 364           proto = s->in2out.port;
 365           r_port = 0;
 366           l_port = 0;
 367         }
 368       else
 369         {
 370           proto = nat_proto_to_ip_proto (s->nat_proto);
 371           l_port = s->in2out.port;
 372           r_port = s->ext_host_port;
 373         }
 374
 375       l_addr = &s->in2out.addr;
 376       r_addr = &s->ext_host_addr;
 377       fib_index = 0;
 378       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 379
 380       if (PREDICT_FALSE
 381           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 382         nat_elog_warn ("in2out_ed key del failed");
 383       return;
 384     }
 385
 386   /* session lookup tables */
 387   if (is_affinity_sessions (s))
 388     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 389                          s->nat_proto, s->out2in.port);
 390   l_addr = &s->out2in.addr;
 391   r_addr = &s->ext_host_addr;
 392   fib_index = s->out2in.fib_index;
 393   if (snat_is_unk_proto_session (s))
 394     {
 395       proto = s->in2out.port;
 396       r_port = 0;
 397       l_port = 0;
 398     }
 399   else
 400     {
 401       proto = nat_proto_to_ip_proto (s->nat_proto);
 402       l_port = s->out2in.port;
 403       r_port = s->ext_host_port;
 404     }
 405   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 406
 407   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0)))
 408     nat_elog_warn ("out2in_ed key del failed");
 409
 410   l_addr = &s->in2out.addr;
 411   fib_index = s->in2out.fib_index;
 412
 413   if (!snat_is_unk_proto_session (s))
 414     l_port = s->in2out.port;
 415
 416   if (is_twice_nat_session (s))
 417     {
 418       r_addr = &s->ext_host_nat_addr;
 419       r_port = s->ext_host_nat_port;
 420     }
 421   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 422
 423   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 424     nat_elog_warn ("in2out_ed key del failed");
 425
 426   if (!is_ha)
 427     {
 428       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 429                              &s->in2out.addr, s->in2out.port,
 430                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 431                              &s->out2in.addr, s->out2in.port,
 432                              &s->ext_host_addr, s->ext_host_port,
 433                              s->nat_proto, is_twice_nat_session (s));
 434     }
 435
 436   if (snat_is_unk_proto_session (s))
 437     return;
 438
 439   if (!is_ha)
 440     {
 441       snat_ipfix_logging_nat44_ses_delete (thread_index,
 442                                            s->in2out.addr.as_u32,
 443                                            s->out2in.addr.as_u32,
 444                                            s->nat_proto,
 445                                            s->in2out.port,
 446                                            s->out2in.port,
 447                                            s->in2out.fib_index);
 448       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 449                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 450                    thread_index);
 451     }
 452
 453   /* Twice NAT address and port for external host */
 454   if (is_twice_nat_session (s))
 455     {
 456       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 457                                           thread_index,
 458                                           &s->ext_host_nat_addr,
 459                                           s->ext_host_nat_port, s->nat_proto);
 460     }
 461
 462   if (snat_is_session_static (s))
 463     return;
 464
 465   snat_free_outside_address_and_port (sm->addresses, thread_index,
 466                                       &s->out2in.addr, s->out2in.port,
 467                                       s->nat_proto);
 468 }
 469
 470
 471 snat_user_t *
 472 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 473                         u32 thread_index)
 474 {
 475   snat_user_t *u = 0;
 476   snat_user_key_t user_key;
 477   clib_bihash_kv_8_8_t kv, value;
 478   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 479   dlist_elt_t *per_user_list_head_elt;
 480
 481   user_key.addr.as_u32 = addr->as_u32;
 482   user_key.fib_index = fib_index;
 483   kv.key = user_key.as_u64;
 484
 485   /* Ever heard of the "user" = src ip4 address before? */
 486   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 487     {
 488       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 489         {
 490           vlib_increment_simple_counter (&sm->user_limit_reached,
 491                                          thread_index, 0, 1);
 492           nat_elog_warn ("maximum user limit reached");
 493           return NULL;
 494         }
 495       /* no, make a new one */
 496       pool_get (tsm->users, u);
 497       clib_memset (u, 0, sizeof (*u));
 498
 499       u->addr.as_u32 = addr->as_u32;
 500       u->fib_index = fib_index;
 501
 502       pool_get (tsm->list_pool, per_user_list_head_elt);
 503
 504       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 505         tsm->list_pool;
 506
 507       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 508
 509       kv.value = u - tsm->users;
 510
 511       /* add user */
 512       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 513         {
 514           nat_elog_warn ("user_hash key add failed");
 515           nat44_delete_user_with_no_session (sm, u, thread_index);
 516           return NULL;
 517         }
 518
 519       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 520                                pool_elts (tsm->users));
 521     }
 522   else
 523     {
 524       u = pool_elt_at_index (tsm->users, value.value);
 525     }
 526
 527   return u;
 528 }
 529
 530 snat_session_t *
 531 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 532                               u32 thread_index, f64 now)
 533 {
 534   snat_session_t *s;
 535   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 536   u32 oldest_per_user_translation_list_index, session_index;
 537   dlist_elt_t *oldest_per_user_translation_list_elt;
 538   dlist_elt_t *per_user_translation_list_elt;
 539
 540   /* Over quota? Recycle the least recently used translation */
 541   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 542     {
 543       oldest_per_user_translation_list_index =
 544         clib_dlist_remove_head (tsm->list_pool,
 545                                 u->sessions_per_user_list_head_index);
 546
 547       ASSERT (oldest_per_user_translation_list_index != ~0);
 548
 549       /* Add it back to the end of the LRU list */
 550       clib_dlist_addtail (tsm->list_pool,
 551                           u->sessions_per_user_list_head_index,
 552                           oldest_per_user_translation_list_index);
 553       /* Get the list element */
 554       oldest_per_user_translation_list_elt =
 555         pool_elt_at_index (tsm->list_pool,
 556                            oldest_per_user_translation_list_index);
 557
 558       /* Get the session index from the list element */
 559       session_index = oldest_per_user_translation_list_elt->value;
 560
 561       /* Get the session */
 562       s = pool_elt_at_index (tsm->sessions, session_index);
 563       nat_free_session_data (sm, s, thread_index, 0);
 564       if (snat_is_session_static (s))
 565         u->nstaticsessions--;
 566       else
 567         u->nsessions--;
 568       s->flags = 0;
 569       s->total_bytes = 0;
 570       s->total_pkts = 0;
 571       s->state = 0;
 572       s->ext_host_addr.as_u32 = 0;
 573       s->ext_host_port = 0;
 574       s->ext_host_nat_addr.as_u32 = 0;
 575       s->ext_host_nat_port = 0;
 576     }
 577   else
 578     {
 579       pool_get (tsm->sessions, s);
 580       clib_memset (s, 0, sizeof (*s));
 581
 582       /* Create list elts */
 583       pool_get (tsm->list_pool, per_user_translation_list_elt);
 584       clib_dlist_init (tsm->list_pool,
 585                        per_user_translation_list_elt - tsm->list_pool);
 586
 587       per_user_translation_list_elt->value = s - tsm->sessions;
 588       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 589       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 590
 591       clib_dlist_addtail (tsm->list_pool,
 592                           s->per_user_list_head_index,
 593                           per_user_translation_list_elt - tsm->list_pool);
 594
 595       s->user_index = u - tsm->users;
 596       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 597                                pool_elts (tsm->sessions));
 598     }
 599
 600   s->ha_last_refreshed = now;
 601
 602   return s;
 603 }
 604
 605 void
 606 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 607                           int is_add)
 608 {
 609   fib_prefix_t prefix = {
 610     .fp_len = p_len,
 611     .fp_proto = FIB_PROTOCOL_IP4,
 612     .fp_addr = {
 613                 .ip4.as_u32 = addr->as_u32,
 614                 },
 615   };
 616   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 617
 618   if (is_add)
 619     fib_table_entry_update_one_path (fib_index,
 620                                      &prefix,
 621                                      nat_fib_src_low,
 622                                      (FIB_ENTRY_FLAG_CONNECTED |
 623                                       FIB_ENTRY_FLAG_LOCAL |
 624                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 625                                      DPO_PROTO_IP4,
 626                                      NULL,
 627                                      sw_if_index,
 628                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 629   else
 630     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 631 }
 632
 633 int
 634 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 635                   u8 twice_nat)
 636 {
 637   snat_address_t *ap;
 638   snat_interface_t *i;
 639   vlib_thread_main_t *tm = vlib_get_thread_main ();
 640
 641   if (twice_nat && !sm->endpoint_dependent)
 642     return VNET_API_ERROR_FEATURE_DISABLED;
 643
 644   /* Check if address already exists */
 645   /* *INDENT-OFF* */
 646   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 647     {
 648       if (ap->addr.as_u32 == addr->as_u32)
 649         return VNET_API_ERROR_VALUE_EXIST;
 650     }
 651   /* *INDENT-ON* */
 652
 653   if (twice_nat)
 654     vec_add2 (sm->twice_nat_addresses, ap, 1);
 655   else
 656     vec_add2 (sm->addresses, ap, 1);
 657
 658   ap->addr = *addr;
 659   if (vrf_id != ~0)
 660     ap->fib_index =
 661       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 662                                          nat_fib_src_low);
 663   else
 664     ap->fib_index = ~0;
 665 #define _(N, i, n, s) \
 666   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 667   ap->busy_##n##_ports = 0; \
 668   ap->busy_##n##_ports_per_thread = 0;\
 669   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 670   foreach_nat_protocol
 671 #undef _
 672     if (twice_nat)
 673     return 0;
 674
 675   /* Add external address to FIB */
 676   /* *INDENT-OFF* */
 677   pool_foreach (i, sm->interfaces,
 678   ({
 679     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 680       continue;
 681
 682     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 683     break;
 684   }));
 685   pool_foreach (i, sm->output_feature_interfaces,
 686   ({
 687     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 688       continue;
 689
 690     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 691     break;
 692   }));
 693   /* *INDENT-ON* */
 694
 695   return 0;
 696 }
 697
 698 static int
 699 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 700 {
 701   snat_static_mapping_t *m;
 702   /* *INDENT-OFF* */
 703   pool_foreach (m, sm->static_mappings,
 704   ({
 705       if (is_addr_only_static_mapping (m) ||
 706           is_out2in_only_static_mapping (m) ||
 707           is_identity_static_mapping (m))
 708         continue;
 709       if (m->external_addr.as_u32 == addr.as_u32)
 710         return 1;
 711   }));
 712   /* *INDENT-ON* */
 713
 714   return 0;
 715 }
 716
 717 static void
 718 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 719                                        ip4_address_t l_addr,
 720                                        u16 l_port,
 721                                        u32 sw_if_index,
 722                                        u16 e_port,
 723                                        u32 vrf_id,
 724                                        nat_protocol_t proto,
 725                                        int addr_only, int is_add, u8 * tag,
 726                                        int twice_nat, int out2in_only,
 727                                        int identity_nat)
 728 {
 729   snat_static_map_resolve_t *rp;
 730
 731   vec_add2 (sm->to_resolve, rp, 1);
 732   rp->l_addr.as_u32 = l_addr.as_u32;
 733   rp->l_port = l_port;
 734   rp->sw_if_index = sw_if_index;
 735   rp->e_port = e_port;
 736   rp->vrf_id = vrf_id;
 737   rp->proto = proto;
 738   rp->addr_only = addr_only;
 739   rp->is_add = is_add;
 740   rp->twice_nat = twice_nat;
 741   rp->out2in_only = out2in_only;
 742   rp->identity_nat = identity_nat;
 743   rp->tag = vec_dup (tag);
 744 }
 745
 746 static u32
 747 get_thread_idx_by_port (u16 e_port)
 748 {
 749   snat_main_t *sm = &snat_main;
 750   u32 thread_idx = sm->num_workers;
 751   if (sm->num_workers > 1)
 752     {
 753       thread_idx =
 754         sm->first_worker_index +
 755         sm->workers[(e_port - 1024) / sm->port_per_thread];
 756     }
 757   return thread_idx;
 758 }
 759
 760 void
 761 snat_static_mapping_del_sessions (snat_main_t * sm,
 762                                   snat_main_per_thread_data_t * tsm,
 763                                   snat_user_key_t u_key, int addr_only,
 764                                   ip4_address_t e_addr, u16 e_port)
 765 {
 766   clib_bihash_kv_8_8_t kv, value;
 767   kv.key = u_key.as_u64;
 768   u64 user_index;
 769   dlist_elt_t *head, *elt;
 770   snat_user_t *u;
 771   snat_session_t *s;
 772   u32 elt_index, head_index, ses_index;
 773   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 774     {
 775       user_index = value.value;
 776       u = pool_elt_at_index (tsm->users, user_index);
 777       if (u->nstaticsessions)
 778         {
 779           head_index = u->sessions_per_user_list_head_index;
 780           head = pool_elt_at_index (tsm->list_pool, head_index);
 781           elt_index = head->next;
 782           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 783           ses_index = elt->value;
 784           while (ses_index != ~0)
 785             {
 786               s = pool_elt_at_index (tsm->sessions, ses_index);
 787               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 788               ses_index = elt->value;
 789
 790               if (!addr_only)
 791                 {
 792                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 793                       (s->out2in.port != e_port))
 794                     continue;
 795                 }
 796
 797               if (is_lb_session (s))
 798                 continue;
 799
 800               if (!snat_is_session_static (s))
 801                 continue;
 802
 803               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 804               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 805
 806               if (!addr_only)
 807                 break;
 808             }
 809         }
 810     }
 811 }
 812
 813 void
 814 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 815                                      snat_main_per_thread_data_t * tsm,
 816                                      ip4_address_t l_addr,
 817                                      u16 l_port,
 818                                      u8 protocol,
 819                                      u32 fib_index, int addr_only,
 820                                      ip4_address_t e_addr, u16 e_port)
 821 {
 822   snat_session_t *s;
 823   u32 *indexes_to_free = NULL;
 824   /* *INDENT-OFF* */
 825   pool_foreach (s, tsm->sessions, {
 826     if (s->in2out.fib_index != fib_index ||
 827         s->in2out.addr.as_u32 != l_addr.as_u32)
 828       {
 829         continue;
 830       }
 831     if (!addr_only)
 832       {
 833         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 834             s->out2in.port != e_port ||
 835             s->in2out.port != l_port ||
 836             s->nat_proto != protocol)
 837           continue;
 838       }
 839
 840     if (is_lb_session (s))
 841       continue;
 842     if (!snat_is_session_static (s))
 843       continue;
 844     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 845     vec_add1 (indexes_to_free, s - tsm->sessions);
 846     if (!addr_only)
 847       break;
 848   });
 849   /* *INDENT-ON* */
 850   u32 *ses_index;
 851   vec_foreach (ses_index, indexes_to_free)
 852   {
 853     s = pool_elt_at_index (tsm->sessions, *ses_index);
 854     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 855   }
 856   vec_free (indexes_to_free);
 857 }
 858
 859 int
 860 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 861                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 862                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 863                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 864                          u8 identity_nat)
 865 {
 866   snat_main_t *sm = &snat_main;
 867   snat_static_mapping_t *m;
 868   clib_bihash_kv_8_8_t kv, value;
 869   snat_address_t *a = 0;
 870   u32 fib_index = ~0;
 871   snat_interface_t *interface;
 872   int i;
 873   snat_main_per_thread_data_t *tsm;
 874   snat_user_key_t u_key;
 875   snat_user_t *u;
 876   dlist_elt_t *head, *elt;
 877   u32 elt_index, head_index;
 878   u32 ses_index;
 879   u64 user_index;
 880   snat_session_t *s;
 881   snat_static_map_resolve_t *rp, *rp_match = 0;
 882   nat44_lb_addr_port_t *local;
 883   u32 find = ~0;
 884
 885   if (!sm->endpoint_dependent)
 886     {
 887       if (twice_nat || out2in_only)
 888         return VNET_API_ERROR_FEATURE_DISABLED;
 889     }
 890
 891   /* If the external address is a specific interface address */
 892   if (sw_if_index != ~0)
 893     {
 894       ip4_address_t *first_int_addr;
 895
 896       for (i = 0; i < vec_len (sm->to_resolve); i++)
 897         {
 898           rp = sm->to_resolve + i;
 899           if (rp->sw_if_index != sw_if_index ||
 900               rp->l_addr.as_u32 != l_addr.as_u32 ||
 901               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 902             continue;
 903
 904           if (!addr_only)
 905             {
 906               if ((rp->l_port != l_port && rp->e_port != e_port)
 907                   || rp->proto != proto)
 908                 continue;
 909             }
 910
 911           rp_match = rp;
 912           break;
 913         }
 914
 915       /* Might be already set... */
 916       first_int_addr = ip4_interface_first_address
 917         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 918
 919       if (is_add)
 920         {
 921           if (rp_match)
 922             return VNET_API_ERROR_VALUE_EXIST;
 923
 924           snat_add_static_mapping_when_resolved
 925             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 926              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 927
 928           /* DHCP resolution required? */
 929           if (first_int_addr == 0)
 930             {
 931               return 0;
 932             }
 933           else
 934             {
 935               e_addr.as_u32 = first_int_addr->as_u32;
 936               /* Identity mapping? */
 937               if (l_addr.as_u32 == 0)
 938                 l_addr.as_u32 = e_addr.as_u32;
 939             }
 940         }
 941       else
 942         {
 943           if (!rp_match)
 944             return VNET_API_ERROR_NO_SUCH_ENTRY;
 945
 946           vec_del1 (sm->to_resolve, i);
 947
 948           if (first_int_addr)
 949             {
 950               e_addr.as_u32 = first_int_addr->as_u32;
 951               /* Identity mapping? */
 952               if (l_addr.as_u32 == 0)
 953                 l_addr.as_u32 = e_addr.as_u32;
 954             }
 955           else
 956             return 0;
 957         }
 958     }
 959
 960   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 961   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 962     m = 0;
 963   else
 964     m = pool_elt_at_index (sm->static_mappings, value.value);
 965
 966   if (is_add)
 967     {
 968       if (m)
 969         {
 970           if (is_identity_static_mapping (m))
 971             {
 972               /* *INDENT-OFF* */
 973               pool_foreach (local, m->locals,
 974               ({
 975                 if (local->vrf_id == vrf_id)
 976                   return VNET_API_ERROR_VALUE_EXIST;
 977               }));
 978               /* *INDENT-ON* */
 979               pool_get (m->locals, local);
 980               local->vrf_id = vrf_id;
 981               local->fib_index =
 982                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 983                                                    nat_fib_src_low);
 984               init_nat_kv (&kv, m->local_addr, m->local_port,
 985                            local->fib_index, m->proto,
 986                            m - sm->static_mappings);
 987               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 988               return 0;
 989             }
 990           else
 991             return VNET_API_ERROR_VALUE_EXIST;
 992         }
 993
 994       if (twice_nat && addr_only)
 995         return VNET_API_ERROR_UNSUPPORTED;
 996
 997       /* Convert VRF id to FIB index */
 998       if (vrf_id != ~0)
 999         fib_index =
1000           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1001                                              nat_fib_src_low);
1002       /* If not specified use inside VRF id from SNAT plugin startup config */
1003       else
1004         {
1005           fib_index = sm->inside_fib_index;
1006           vrf_id = sm->inside_vrf_id;
1007           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1008         }
1009
1010       if (!(out2in_only || identity_nat))
1011         {
1012           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
1013                       addr_only ? 0 : proto);
1014           if (!clib_bihash_search_8_8
1015               (&sm->static_mapping_by_local, &kv, &value))
1016             return VNET_API_ERROR_VALUE_EXIST;
1017         }
1018
1019       /* Find external address in allocated addresses and reserve port for
1020          address and port pair mapping when dynamic translations enabled */
1021       if (!(addr_only || sm->static_mapping_only || out2in_only))
1022         {
1023           for (i = 0; i < vec_len (sm->addresses); i++)
1024             {
1025               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1026                 {
1027                   a = sm->addresses + i;
1028                   /* External port must be unused */
1029                   switch (proto)
1030                     {
1031 #define _(N, j, n, s) \
1032                     case NAT_PROTOCOL_##N: \
1033                       if (a->busy_##n##_port_refcounts[e_port]) \
1034                         return VNET_API_ERROR_INVALID_VALUE; \
1035                       ++a->busy_##n##_port_refcounts[e_port]; \
1036                       if (e_port > 1024) \
1037                         { \
1038                           a->busy_##n##_ports++; \
1039                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1040                         } \
1041                       break;
1042                       foreach_nat_protocol
1043 #undef _
1044                     default:
1045                       nat_elog_info ("unknown protocol");
1046                       return VNET_API_ERROR_INVALID_VALUE_2;
1047                     }
1048                   break;
1049                 }
1050             }
1051           /* External address must be allocated */
1052           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1053             {
1054               if (sw_if_index != ~0)
1055                 {
1056                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1057                     {
1058                       rp = sm->to_resolve + i;
1059                       if (rp->addr_only)
1060                         continue;
1061                       if (rp->sw_if_index != sw_if_index &&
1062                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1063                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1064                           rp->e_port != e_port && rp->proto != proto)
1065                         continue;
1066
1067                       vec_del1 (sm->to_resolve, i);
1068                       break;
1069                     }
1070                 }
1071               return VNET_API_ERROR_NO_SUCH_ENTRY;
1072             }
1073         }
1074
1075       pool_get (sm->static_mappings, m);
1076       clib_memset (m, 0, sizeof (*m));
1077       m->tag = vec_dup (tag);
1078       m->local_addr = l_addr;
1079       m->external_addr = e_addr;
1080       m->twice_nat = twice_nat;
1081       if (out2in_only)
1082         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1083       if (addr_only)
1084         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1085       if (identity_nat)
1086         {
1087           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1088           pool_get (m->locals, local);
1089           local->vrf_id = vrf_id;
1090           local->fib_index = fib_index;
1091         }
1092       else
1093         {
1094           m->vrf_id = vrf_id;
1095           m->fib_index = fib_index;
1096         }
1097       if (!addr_only)
1098         {
1099           m->local_port = l_port;
1100           m->external_port = e_port;
1101           m->proto = proto;
1102         }
1103
1104       if (sm->num_workers > 1)
1105         {
1106           ip4_header_t ip = {
1107             .src_address = m->local_addr,
1108           };
1109           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1110           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1111         }
1112       else
1113         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1114
1115       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
1116                    m - sm->static_mappings);
1117       if (!out2in_only)
1118         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1119
1120       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1121                    m - sm->static_mappings);
1122       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1123
1124       /* Delete dynamic sessions matching local address (+ local port) */
1125       if (!(sm->static_mapping_only))
1126         {
1127           u_key.addr = m->local_addr;
1128           u_key.fib_index = m->fib_index;
1129           kv.key = u_key.as_u64;
1130           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1131             {
1132               user_index = value.value;
1133               u = pool_elt_at_index (tsm->users, user_index);
1134               if (u->nsessions)
1135                 {
1136                   head_index = u->sessions_per_user_list_head_index;
1137                   head = pool_elt_at_index (tsm->list_pool, head_index);
1138                   elt_index = head->next;
1139                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1140                   ses_index = elt->value;
1141                   while (ses_index != ~0)
1142                     {
1143                       s = pool_elt_at_index (tsm->sessions, ses_index);
1144                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1145                       ses_index = elt->value;
1146
1147                       if (snat_is_session_static (s))
1148                         continue;
1149
1150                       if (!addr_only && s->in2out.port != m->local_port)
1151                         continue;
1152
1153                       nat_free_session_data (sm, s,
1154                                              tsm - sm->per_thread_data, 0);
1155                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1156
1157                       if (!addr_only && !sm->endpoint_dependent)
1158                         break;
1159                     }
1160                 }
1161             }
1162         }
1163     }
1164   else
1165     {
1166       if (!m)
1167         {
1168           if (sw_if_index != ~0)
1169             return 0;
1170           else
1171             return VNET_API_ERROR_NO_SUCH_ENTRY;
1172         }
1173
1174       if (identity_nat)
1175         {
1176           if (vrf_id == ~0)
1177             vrf_id = sm->inside_vrf_id;
1178
1179           /* *INDENT-OFF* */
1180           pool_foreach (local, m->locals,
1181           ({
1182             if (local->vrf_id == vrf_id)
1183               find = local - m->locals;
1184           }));
1185           /* *INDENT-ON* */
1186           if (find == ~0)
1187             return VNET_API_ERROR_NO_SUCH_ENTRY;
1188
1189           local = pool_elt_at_index (m->locals, find);
1190           fib_index = local->fib_index;
1191           pool_put (m->locals, local);
1192         }
1193       else
1194         fib_index = m->fib_index;
1195
1196       /* Free external address port */
1197       if (!(addr_only || sm->static_mapping_only || out2in_only))
1198         {
1199           for (i = 0; i < vec_len (sm->addresses); i++)
1200             {
1201               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1202                 {
1203                   a = sm->addresses + i;
1204                   switch (proto)
1205                     {
1206 #define _(N, j, n, s) \
1207                     case NAT_PROTOCOL_##N: \
1208                       --a->busy_##n##_port_refcounts[e_port]; \
1209                       if (e_port > 1024) \
1210                         { \
1211                           a->busy_##n##_ports--; \
1212                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1213                         } \
1214                       break;
1215                       foreach_nat_protocol
1216 #undef _
1217                     default:
1218                       nat_elog_info ("unknown protocol");
1219                       return VNET_API_ERROR_INVALID_VALUE_2;
1220                     }
1221                   break;
1222                 }
1223             }
1224         }
1225
1226       if (sm->num_workers > 1)
1227         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1228       else
1229         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1230
1231       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1232       if (!out2in_only)
1233         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1234
1235       /* Delete session(s) for static mapping if exist */
1236       if (!(sm->static_mapping_only) ||
1237           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1238         {
1239           if (sm->endpoint_dependent)
1240             {
1241               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1242                                                    m->local_port, m->proto,
1243                                                    fib_index, addr_only,
1244                                                    e_addr, e_port);
1245             }
1246           else
1247             {
1248               u_key.addr = m->local_addr;
1249               u_key.fib_index = fib_index;
1250               kv.key = u_key.as_u64;
1251               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1252                                                 e_addr, e_port);
1253             }
1254         }
1255
1256       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1257       if (pool_elts (m->locals))
1258         return 0;
1259
1260       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1261       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1262
1263       vec_free (m->tag);
1264       vec_free (m->workers);
1265       /* Delete static mapping from pool */
1266       pool_put (sm->static_mappings, m);
1267     }
1268
1269   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1270     return 0;
1271
1272   /* Add/delete external address to FIB */
1273   /* *INDENT-OFF* */
1274   pool_foreach (interface, sm->interfaces,
1275   ({
1276     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1277       continue;
1278
1279     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1280     break;
1281   }));
1282   pool_foreach (interface, sm->output_feature_interfaces,
1283   ({
1284     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1285       continue;
1286
1287     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1288     break;
1289   }));
1290   /* *INDENT-ON* */
1291
1292   return 0;
1293 }
1294
1295 int
1296 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1297                                  nat_protocol_t proto,
1298                                  nat44_lb_addr_port_t * locals, u8 is_add,
1299                                  twice_nat_type_t twice_nat, u8 out2in_only,
1300                                  u8 * tag, u32 affinity)
1301 {
1302   snat_main_t *sm = &snat_main;
1303   snat_static_mapping_t *m;
1304   clib_bihash_kv_8_8_t kv, value;
1305   snat_address_t *a = 0;
1306   int i;
1307   nat44_lb_addr_port_t *local;
1308   snat_main_per_thread_data_t *tsm;
1309   snat_session_t *s;
1310   uword *bitmap = 0;
1311
1312   if (!sm->endpoint_dependent)
1313     return VNET_API_ERROR_FEATURE_DISABLED;
1314
1315   init_nat_k (&kv, e_addr, e_port, 0, proto);
1316   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1317     m = 0;
1318   else
1319     m = pool_elt_at_index (sm->static_mappings, value.value);
1320
1321   if (is_add)
1322     {
1323       if (m)
1324         return VNET_API_ERROR_VALUE_EXIST;
1325
1326       if (vec_len (locals) < 2)
1327         return VNET_API_ERROR_INVALID_VALUE;
1328
1329       /* Find external address in allocated addresses and reserve port for
1330          address and port pair mapping when dynamic translations enabled */
1331       if (!(sm->static_mapping_only || out2in_only))
1332         {
1333           for (i = 0; i < vec_len (sm->addresses); i++)
1334             {
1335               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1336                 {
1337                   a = sm->addresses + i;
1338                   /* External port must be unused */
1339                   switch (proto)
1340                     {
1341 #define _(N, j, n, s) \
1342                     case NAT_PROTOCOL_##N: \
1343                       if (a->busy_##n##_port_refcounts[e_port]) \
1344                         return VNET_API_ERROR_INVALID_VALUE; \
1345                       ++a->busy_##n##_port_refcounts[e_port]; \
1346                       if (e_port > 1024) \
1347                         { \
1348                           a->busy_##n##_ports++; \
1349                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1350                         } \
1351                       break;
1352                       foreach_nat_protocol
1353 #undef _
1354                     default:
1355                       nat_elog_info ("unknown protocol");
1356                       return VNET_API_ERROR_INVALID_VALUE_2;
1357                     }
1358                   break;
1359                 }
1360             }
1361           /* External address must be allocated */
1362           if (!a)
1363             return VNET_API_ERROR_NO_SUCH_ENTRY;
1364         }
1365
1366       pool_get (sm->static_mappings, m);
1367       clib_memset (m, 0, sizeof (*m));
1368       m->tag = vec_dup (tag);
1369       m->external_addr = e_addr;
1370       m->external_port = e_port;
1371       m->proto = proto;
1372       m->twice_nat = twice_nat;
1373       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1374       if (out2in_only)
1375         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1376       m->affinity = affinity;
1377
1378       if (affinity)
1379         m->affinity_per_service_list_head_index =
1380           nat_affinity_get_per_service_list_head_index ();
1381       else
1382         m->affinity_per_service_list_head_index = ~0;
1383
1384       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1385                    m - sm->static_mappings);
1386       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1387         {
1388           nat_elog_err ("static_mapping_by_external key add failed");
1389           return VNET_API_ERROR_UNSPECIFIED;
1390         }
1391
1392       for (i = 0; i < vec_len (locals); i++)
1393         {
1394           locals[i].fib_index =
1395             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1396                                                locals[i].vrf_id,
1397                                                nat_fib_src_low);
1398           if (!out2in_only)
1399             {
1400               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1401                            locals[i].fib_index, m->proto,
1402                            m - sm->static_mappings);
1403               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1404             }
1405           locals[i].prefix = (i == 0) ? locals[i].probability :
1406             (locals[i - 1].prefix + locals[i].probability);
1407           pool_get (m->locals, local);
1408           *local = locals[i];
1409           if (sm->num_workers > 1)
1410             {
1411               ip4_header_t ip = {
1412                 .src_address = locals[i].addr,
1413               };
1414               bitmap =
1415                 clib_bitmap_set (bitmap,
1416                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1417                                  1);
1418             }
1419         }
1420
1421       /* Assign workers */
1422       if (sm->num_workers > 1)
1423         {
1424           /* *INDENT-OFF* */
1425           clib_bitmap_foreach (i, bitmap,
1426             ({
1427                vec_add1(m->workers, i);
1428             }));
1429           /* *INDENT-ON* */
1430         }
1431     }
1432   else
1433     {
1434       if (!m)
1435         return VNET_API_ERROR_NO_SUCH_ENTRY;
1436
1437       if (!is_lb_static_mapping (m))
1438         return VNET_API_ERROR_INVALID_VALUE;
1439
1440       /* Free external address port */
1441       if (!(sm->static_mapping_only || out2in_only))
1442         {
1443           for (i = 0; i < vec_len (sm->addresses); i++)
1444             {
1445               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1446                 {
1447                   a = sm->addresses + i;
1448                   switch (proto)
1449                     {
1450 #define _(N, j, n, s) \
1451                     case NAT_PROTOCOL_##N: \
1452                       --a->busy_##n##_port_refcounts[e_port]; \
1453                       if (e_port > 1024) \
1454                         { \
1455                           a->busy_##n##_ports--; \
1456                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1457                         } \
1458                       break;
1459                       foreach_nat_protocol
1460 #undef _
1461                     default:
1462                       nat_elog_info ("unknown protocol");
1463                       return VNET_API_ERROR_INVALID_VALUE_2;
1464                     }
1465                   break;
1466                 }
1467             }
1468         }
1469
1470       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1471       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1472         {
1473           nat_elog_err ("static_mapping_by_external key del failed");
1474           return VNET_API_ERROR_UNSPECIFIED;
1475         }
1476
1477       /* *INDENT-OFF* */
1478       pool_foreach (local, m->locals,
1479       ({
1480           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1481                             nat_fib_src_low);
1482           if (!out2in_only)
1483             {
1484 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1485               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1486                 {
1487                   nat_elog_err ("static_mapping_by_local key del failed");
1488                   return VNET_API_ERROR_UNSPECIFIED;
1489                 }
1490             }
1491
1492           if (sm->num_workers > 1)
1493             {
1494               ip4_header_t ip = {
1495                 .src_address = local->addr,
1496               };
1497               tsm = vec_elt_at_index (sm->per_thread_data,
1498                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1499             }
1500           else
1501             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1502
1503           /* Delete sessions */
1504           pool_foreach (s, tsm->sessions, {
1505             if (!(is_lb_session (s)))
1506               continue;
1507
1508             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1509                 s->in2out.port != local->port)
1510               continue;
1511
1512             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1513             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1514           });
1515       }));
1516       /* *INDENT-ON* */
1517       if (m->affinity)
1518         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1519       pool_free (m->locals);
1520       vec_free (m->tag);
1521       vec_free (m->workers);
1522
1523       pool_put (sm->static_mappings, m);
1524     }
1525
1526   return 0;
1527 }
1528
1529 int
1530 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1531                                        ip4_address_t l_addr, u16 l_port,
1532                                        nat_protocol_t proto, u32 vrf_id,
1533                                        u8 probability, u8 is_add)
1534 {
1535   snat_main_t *sm = &snat_main;
1536   snat_static_mapping_t *m = 0;
1537   clib_bihash_kv_8_8_t kv, value;
1538   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1539   snat_main_per_thread_data_t *tsm;
1540   snat_session_t *s;
1541   u32 *locals = 0;
1542   uword *bitmap = 0;
1543   int i;
1544
1545   if (!sm->endpoint_dependent)
1546     return VNET_API_ERROR_FEATURE_DISABLED;
1547
1548   init_nat_k (&kv, e_addr, e_port, 0, proto);
1549   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1550     m = pool_elt_at_index (sm->static_mappings, value.value);
1551
1552   if (!m)
1553     return VNET_API_ERROR_NO_SUCH_ENTRY;
1554
1555   if (!is_lb_static_mapping (m))
1556     return VNET_API_ERROR_INVALID_VALUE;
1557
1558   /* *INDENT-OFF* */
1559   pool_foreach (local, m->locals,
1560   ({
1561     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1562         (local->vrf_id == vrf_id))
1563       {
1564         match_local = local;
1565         break;
1566       }
1567   }));
1568   /* *INDENT-ON* */
1569
1570   if (is_add)
1571     {
1572       if (match_local)
1573         return VNET_API_ERROR_VALUE_EXIST;
1574
1575       pool_get (m->locals, local);
1576       clib_memset (local, 0, sizeof (*local));
1577       local->addr.as_u32 = l_addr.as_u32;
1578       local->port = l_port;
1579       local->probability = probability;
1580       local->vrf_id = vrf_id;
1581       local->fib_index =
1582         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1583                                            nat_fib_src_low);
1584
1585       if (!is_out2in_only_static_mapping (m))
1586         {
1587           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto,
1588                        m - sm->static_mappings);
1589           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1590             nat_elog_err ("static_mapping_by_local key add failed");
1591         }
1592     }
1593   else
1594     {
1595       if (!match_local)
1596         return VNET_API_ERROR_NO_SUCH_ENTRY;
1597
1598       if (pool_elts (m->locals) < 3)
1599         return VNET_API_ERROR_UNSPECIFIED;
1600
1601       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1602                         nat_fib_src_low);
1603
1604       if (!is_out2in_only_static_mapping (m))
1605         {
1606           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1607           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1608             nat_elog_err ("static_mapping_by_local key del failed");
1609         }
1610
1611       if (sm->num_workers > 1)
1612         {
1613           ip4_header_t ip = {
1614             .src_address = local->addr,
1615           };
1616           tsm = vec_elt_at_index (sm->per_thread_data,
1617                                   sm->worker_in2out_cb (&ip, m->fib_index,
1618                                                         0));
1619         }
1620       else
1621         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1622
1623       /* Delete sessions */
1624       /* *INDENT-OFF* */
1625       pool_foreach (s, tsm->sessions, {
1626         if (!(is_lb_session (s)))
1627           continue;
1628
1629         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1630             s->in2out.port != match_local->port)
1631           continue;
1632
1633         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1634         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1635       });
1636       /* *INDENT-ON* */
1637
1638       pool_put (m->locals, match_local);
1639     }
1640
1641   vec_free (m->workers);
1642
1643   /* *INDENT-OFF* */
1644   pool_foreach (local, m->locals,
1645   ({
1646     vec_add1 (locals, local - m->locals);
1647     if (sm->num_workers > 1)
1648       {
1649         ip4_header_t ip;
1650         ip.src_address.as_u32 = local->addr.as_u32,
1651         bitmap = clib_bitmap_set (bitmap,
1652                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1653                                   1);
1654       }
1655   }));
1656   /* *INDENT-ON* */
1657
1658   ASSERT (vec_len (locals) > 1);
1659
1660   local = pool_elt_at_index (m->locals, locals[0]);
1661   local->prefix = local->probability;
1662   for (i = 1; i < vec_len (locals); i++)
1663     {
1664       local = pool_elt_at_index (m->locals, locals[i]);
1665       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1666       local->prefix = local->probability + prev_local->prefix;
1667     }
1668
1669   /* Assign workers */
1670   if (sm->num_workers > 1)
1671     {
1672       /* *INDENT-OFF* */
1673       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1674       /* *INDENT-ON* */
1675     }
1676
1677   return 0;
1678 }
1679
1680 int
1681 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1682                   u8 twice_nat)
1683 {
1684   snat_address_t *a = 0;
1685   snat_session_t *ses;
1686   u32 *ses_to_be_removed = 0, *ses_index;
1687   snat_main_per_thread_data_t *tsm;
1688   snat_static_mapping_t *m;
1689   snat_interface_t *interface;
1690   int i;
1691   snat_address_t *addresses =
1692     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1693
1694   /* Find SNAT address */
1695   for (i = 0; i < vec_len (addresses); i++)
1696     {
1697       if (addresses[i].addr.as_u32 == addr.as_u32)
1698         {
1699           a = addresses + i;
1700           break;
1701         }
1702     }
1703   if (!a)
1704     return VNET_API_ERROR_NO_SUCH_ENTRY;
1705
1706   if (delete_sm)
1707     {
1708       /* *INDENT-OFF* */
1709       pool_foreach (m, sm->static_mappings,
1710       ({
1711           if (m->external_addr.as_u32 == addr.as_u32)
1712             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1713                                             m->local_port, m->external_port,
1714                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1715                                             m->proto, 0, m->twice_nat,
1716                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1717       }));
1718       /* *INDENT-ON* */
1719     }
1720   else
1721     {
1722       /* Check if address is used in some static mapping */
1723       if (is_snat_address_used_in_static_mapping (sm, addr))
1724         {
1725           nat_elog_notice ("address used in static mapping");
1726           return VNET_API_ERROR_UNSPECIFIED;
1727         }
1728     }
1729
1730   if (a->fib_index != ~0)
1731     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1732
1733   /* Delete sessions using address */
1734   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1735     {
1736       /* *INDENT-OFF* */
1737       vec_foreach (tsm, sm->per_thread_data)
1738         {
1739           pool_foreach (ses, tsm->sessions, ({
1740             if (ses->out2in.addr.as_u32 == addr.as_u32)
1741               {
1742                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1743                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1744               }
1745           }));
1746
1747           if (sm->endpoint_dependent){
1748               vec_foreach (ses_index, ses_to_be_removed)
1749                 {
1750                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1751                   nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1752                 }
1753           }else{
1754               vec_foreach (ses_index, ses_to_be_removed)
1755                 {
1756                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1757                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1758                 }
1759           }
1760
1761           vec_free (ses_to_be_removed);
1762         }
1763       /* *INDENT-ON* */
1764     }
1765
1766 #define _(N, i, n, s) \
1767   vec_free (a->busy_##n##_ports_per_thread);
1768   foreach_nat_protocol
1769 #undef _
1770     if (twice_nat)
1771     {
1772       vec_del1 (sm->twice_nat_addresses, i);
1773       return 0;
1774     }
1775   else
1776     vec_del1 (sm->addresses, i);
1777
1778   /* Delete external address from FIB */
1779   /* *INDENT-OFF* */
1780   pool_foreach (interface, sm->interfaces,
1781   ({
1782     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1783       continue;
1784
1785     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1786     break;
1787   }));
1788   pool_foreach (interface, sm->output_feature_interfaces,
1789   ({
1790     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1791       continue;
1792
1793     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1794     break;
1795   }));
1796   /* *INDENT-ON* */
1797
1798   return 0;
1799 }
1800
1801 int
1802 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1803 {
1804   snat_main_t *sm = &snat_main;
1805   snat_interface_t *i;
1806   const char *feature_name, *del_feature_name;
1807   snat_address_t *ap;
1808   snat_static_mapping_t *m;
1809   snat_det_map_t *dm;
1810   nat_outside_fib_t *outside_fib;
1811   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1812                                                        sw_if_index);
1813
1814   if (sm->out2in_dpo && !is_inside)
1815     return VNET_API_ERROR_UNSUPPORTED;
1816
1817   /* *INDENT-OFF* */
1818   pool_foreach (i, sm->output_feature_interfaces,
1819   ({
1820     if (i->sw_if_index == sw_if_index)
1821       return VNET_API_ERROR_VALUE_EXIST;
1822   }));
1823   /* *INDENT-ON* */
1824
1825   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1826     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1827   else
1828     {
1829       if (sm->num_workers > 1 && !sm->deterministic)
1830         feature_name =
1831           is_inside ? "nat44-in2out-worker-handoff" :
1832           "nat44-out2in-worker-handoff";
1833       else if (sm->deterministic)
1834         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1835       else if (sm->endpoint_dependent)
1836         {
1837           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1838         }
1839       else
1840         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1841     }
1842
1843   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1844     sm->fq_in2out_index =
1845       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1846
1847   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1848     sm->fq_out2in_index =
1849       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1850
1851   if (!is_inside)
1852     {
1853       /* *INDENT-OFF* */
1854       vec_foreach (outside_fib, sm->outside_fibs)
1855         {
1856           if (outside_fib->fib_index == fib_index)
1857             {
1858               if (is_del)
1859                 {
1860                   outside_fib->refcount--;
1861                   if (!outside_fib->refcount)
1862                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1863                 }
1864               else
1865                 outside_fib->refcount++;
1866               goto feature_set;
1867             }
1868         }
1869       /* *INDENT-ON* */
1870       if (!is_del)
1871         {
1872           vec_add2 (sm->outside_fibs, outside_fib, 1);
1873           outside_fib->refcount = 1;
1874           outside_fib->fib_index = fib_index;
1875         }
1876     }
1877 feature_set:
1878   /* *INDENT-OFF* */
1879   pool_foreach (i, sm->interfaces,
1880   ({
1881     if (i->sw_if_index == sw_if_index)
1882       {
1883         if (is_del)
1884           {
1885             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1886               {
1887                 if (is_inside)
1888                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1889                 else
1890                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1891
1892                 if (sm->num_workers > 1 && !sm->deterministic)
1893                   {
1894                     del_feature_name = "nat44-handoff-classify";
1895                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1896                                                  "nat44-out2in-worker-handoff";
1897                   }
1898                 else if (sm->deterministic)
1899                   {
1900                     del_feature_name = "nat44-det-classify";
1901                     feature_name = !is_inside ?  "nat44-det-in2out" :
1902                                                  "nat44-det-out2in";
1903                   }
1904                 else if (sm->endpoint_dependent)
1905                   {
1906                     del_feature_name = "nat44-ed-classify";
1907                     feature_name = !is_inside ?  "nat-pre-in2out" :
1908                                                  "nat-pre-out2in";
1909                   }
1910                 else
1911                   {
1912                     del_feature_name = "nat44-classify";
1913                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1914                   }
1915
1916                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1917                 if (rv)
1918                   return rv;
1919                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1920                                              sw_if_index, 0, 0, 0);
1921                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1922                                              sw_if_index, 1, 0, 0);
1923                 if (!is_inside)
1924                   {
1925                     if (sm->endpoint_dependent)
1926                       vnet_feature_enable_disable ("ip4-local",
1927                                                    "nat44-ed-hairpinning",
1928                                                    sw_if_index, 1, 0, 0);
1929                     else if (!sm->deterministic)
1930                       vnet_feature_enable_disable ("ip4-local",
1931                                                    "nat44-hairpinning",
1932                                                    sw_if_index, 1, 0, 0);
1933                   }
1934               }
1935             else
1936               {
1937                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1938                 if (rv)
1939                   return rv;
1940                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1941                                              sw_if_index, 0, 0, 0);
1942                 pool_put (sm->interfaces, i);
1943                 if (is_inside)
1944                   {
1945                     if (sm->endpoint_dependent)
1946                       vnet_feature_enable_disable ("ip4-local",
1947                                                    "nat44-ed-hairpinning",
1948                                                    sw_if_index, 0, 0, 0);
1949                     else if (!sm->deterministic)
1950                       vnet_feature_enable_disable ("ip4-local",
1951                                                    "nat44-hairpinning",
1952                                                    sw_if_index, 0, 0, 0);
1953                   }
1954               }
1955           }
1956         else
1957           {
1958             if ((nat_interface_is_inside(i) && is_inside) ||
1959                 (nat_interface_is_outside(i) && !is_inside))
1960               return 0;
1961
1962             if (sm->num_workers > 1 && !sm->deterministic)
1963               {
1964                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1965                                                  "nat44-out2in-worker-handoff";
1966                 feature_name = "nat44-handoff-classify";
1967               }
1968             else if (sm->deterministic)
1969               {
1970                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1971                                                  "nat44-det-out2in";
1972                 feature_name = "nat44-det-classify";
1973               }
1974             else if (sm->endpoint_dependent)
1975               {
1976                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
1977                                                  "nat-pre-out2in";
1978
1979                 feature_name = "nat44-ed-classify";
1980               }
1981             else
1982               {
1983                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1984                 feature_name = "nat44-classify";
1985               }
1986
1987             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1988             if (rv)
1989               return rv;
1990             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1991                                          sw_if_index, 0, 0, 0);
1992             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1993                                          sw_if_index, 1, 0, 0);
1994             if (!is_inside)
1995               {
1996                 if (sm->endpoint_dependent)
1997                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1998                                                sw_if_index, 0, 0, 0);
1999                 else if (!sm->deterministic)
2000                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2001                                                sw_if_index, 0, 0, 0);
2002               }
2003             goto set_flags;
2004           }
2005
2006         goto fib;
2007       }
2008   }));
2009   /* *INDENT-ON* */
2010
2011   if (is_del)
2012     return VNET_API_ERROR_NO_SUCH_ENTRY;
2013
2014   pool_get (sm->interfaces, i);
2015   i->sw_if_index = sw_if_index;
2016   i->flags = 0;
2017   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2018                                0);
2019
2020   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2021   if (rv)
2022     return rv;
2023
2024   if (is_inside && !sm->out2in_dpo)
2025     {
2026       if (sm->endpoint_dependent)
2027         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2028                                      sw_if_index, 1, 0, 0);
2029       else if (!sm->deterministic)
2030         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2031                                      sw_if_index, 1, 0, 0);
2032     }
2033
2034 set_flags:
2035   if (is_inside)
2036     {
2037       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2038       return 0;
2039     }
2040   else
2041     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2042
2043   /* Add/delete external addresses to FIB */
2044 fib:
2045   /* *INDENT-OFF* */
2046   vec_foreach (ap, sm->addresses)
2047     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2048
2049   pool_foreach (m, sm->static_mappings,
2050   ({
2051     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2052       continue;
2053
2054     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2055   }));
2056
2057   pool_foreach (dm, sm->det_maps,
2058   ({
2059     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2060   }));
2061   /* *INDENT-ON* */
2062
2063   return 0;
2064 }
2065
2066 int
2067 snat_interface_add_del_output_feature (u32 sw_if_index,
2068                                        u8 is_inside, int is_del)
2069 {
2070   snat_main_t *sm = &snat_main;
2071   snat_interface_t *i;
2072   snat_address_t *ap;
2073   snat_static_mapping_t *m;
2074   nat_outside_fib_t *outside_fib;
2075   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2076                                                        sw_if_index);
2077
2078
2079   if (sm->deterministic ||
2080       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2081     return VNET_API_ERROR_UNSUPPORTED;
2082
2083   /* *INDENT-OFF* */
2084   pool_foreach (i, sm->interfaces,
2085   ({
2086     if (i->sw_if_index == sw_if_index)
2087       return VNET_API_ERROR_VALUE_EXIST;
2088   }));
2089   /* *INDENT-ON* */
2090
2091   if (!is_inside)
2092     {
2093       /* *INDENT-OFF* */
2094       vec_foreach (outside_fib, sm->outside_fibs)
2095         {
2096           if (outside_fib->fib_index == fib_index)
2097             {
2098               if (is_del)
2099                 {
2100                   outside_fib->refcount--;
2101                   if (!outside_fib->refcount)
2102                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2103                 }
2104               else
2105                 outside_fib->refcount++;
2106               goto feature_set;
2107             }
2108         }
2109       /* *INDENT-ON* */
2110       if (!is_del)
2111         {
2112           vec_add2 (sm->outside_fibs, outside_fib, 1);
2113           outside_fib->refcount = 1;
2114           outside_fib->fib_index = fib_index;
2115         }
2116     }
2117
2118 feature_set:
2119   if (is_inside)
2120     {
2121       if (sm->endpoint_dependent)
2122         {
2123           int rv =
2124             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2125           if (rv)
2126             return rv;
2127           rv =
2128             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2129                                                             !is_del);
2130           if (rv)
2131             return rv;
2132           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2133                                        sw_if_index, !is_del, 0, 0);
2134           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2135                                        sw_if_index, !is_del, 0, 0);
2136         }
2137       else
2138         {
2139           int rv =
2140             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2141           if (rv)
2142             return rv;
2143           rv =
2144             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2145                                                             !is_del);
2146           if (rv)
2147             return rv;
2148           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2149                                        sw_if_index, !is_del, 0, 0);
2150           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2151                                        sw_if_index, !is_del, 0, 0);
2152         }
2153       goto fq;
2154     }
2155
2156   if (sm->num_workers > 1)
2157     {
2158       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2159       if (rv)
2160         return rv;
2161       rv =
2162         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2163       if (rv)
2164         return rv;
2165       vnet_feature_enable_disable ("ip4-unicast",
2166                                    "nat44-out2in-worker-handoff",
2167                                    sw_if_index, !is_del, 0, 0);
2168       vnet_feature_enable_disable ("ip4-output",
2169                                    "nat44-in2out-output-worker-handoff",
2170                                    sw_if_index, !is_del, 0, 0);
2171     }
2172   else
2173     {
2174       if (sm->endpoint_dependent)
2175         {
2176           int rv =
2177             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2178           if (rv)
2179             return rv;
2180           rv =
2181             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2182                                                             !is_del);
2183           if (rv)
2184             return rv;
2185           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2186                                        sw_if_index, !is_del, 0, 0);
2187           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2188                                        sw_if_index, !is_del, 0, 0);
2189         }
2190       else
2191         {
2192           int rv =
2193             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2194           if (rv)
2195             return rv;
2196           rv =
2197             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2198                                                             !is_del);
2199           if (rv)
2200             return rv;
2201           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2202                                        sw_if_index, !is_del, 0, 0);
2203           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2204                                        sw_if_index, !is_del, 0, 0);
2205         }
2206     }
2207
2208 fq:
2209   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2210     sm->fq_in2out_output_index =
2211       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2212
2213   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2214     sm->fq_out2in_index =
2215       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2216
2217   /* *INDENT-OFF* */
2218   pool_foreach (i, sm->output_feature_interfaces,
2219   ({
2220     if (i->sw_if_index == sw_if_index)
2221       {
2222         if (is_del)
2223           pool_put (sm->output_feature_interfaces, i);
2224         else
2225           return VNET_API_ERROR_VALUE_EXIST;
2226
2227         goto fib;
2228       }
2229   }));
2230   /* *INDENT-ON* */
2231
2232   if (is_del)
2233     return VNET_API_ERROR_NO_SUCH_ENTRY;
2234
2235   pool_get (sm->output_feature_interfaces, i);
2236   i->sw_if_index = sw_if_index;
2237   i->flags = 0;
2238   if (is_inside)
2239     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2240   else
2241     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2242
2243   /* Add/delete external addresses to FIB */
2244 fib:
2245   if (is_inside)
2246     return 0;
2247
2248   /* *INDENT-OFF* */
2249   vec_foreach (ap, sm->addresses)
2250     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2251
2252   pool_foreach (m, sm->static_mappings,
2253   ({
2254     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2255       continue;
2256
2257     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2258   }));
2259   /* *INDENT-ON* */
2260
2261   return 0;
2262 }
2263
2264 int
2265 snat_set_workers (uword * bitmap)
2266 {
2267   snat_main_t *sm = &snat_main;
2268   int i, j = 0;
2269
2270   if (sm->num_workers < 2)
2271     return VNET_API_ERROR_FEATURE_DISABLED;
2272
2273   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2274     return VNET_API_ERROR_INVALID_WORKER;
2275
2276   vec_free (sm->workers);
2277   /* *INDENT-OFF* */
2278   clib_bitmap_foreach (i, bitmap,
2279     ({
2280       vec_add1(sm->workers, i);
2281       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2282       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2283       j++;
2284     }));
2285   /* *INDENT-ON* */
2286
2287   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2288
2289   return 0;
2290 }
2291
2292 static void
2293 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2294                          u32 old_fib_index)
2295 {
2296   snat_main_t *sm = &snat_main;
2297   nat_outside_fib_t *outside_fib;
2298   snat_interface_t *i;
2299   u8 is_add = 1;
2300   u8 match = 0;
2301
2302   if (new_fib_index == old_fib_index)
2303     return;
2304
2305   if (!vec_len (sm->outside_fibs))
2306     return;
2307
2308   /* *INDENT-OFF* */
2309   pool_foreach (i, sm->interfaces,
2310     ({
2311       if (i->sw_if_index == sw_if_index)
2312         {
2313           if (!(nat_interface_is_outside (i)))
2314             return;
2315           match = 1;
2316         }
2317     }));
2318
2319   pool_foreach (i, sm->output_feature_interfaces,
2320     ({
2321       if (i->sw_if_index == sw_if_index)
2322         {
2323           if (!(nat_interface_is_outside (i)))
2324             return;
2325           match = 1;
2326         }
2327     }));
2328   /* *INDENT-ON* */
2329
2330   if (!match)
2331     return;
2332
2333   vec_foreach (outside_fib, sm->outside_fibs)
2334   {
2335     if (outside_fib->fib_index == old_fib_index)
2336       {
2337         outside_fib->refcount--;
2338         if (!outside_fib->refcount)
2339           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2340         break;
2341       }
2342   }
2343
2344   vec_foreach (outside_fib, sm->outside_fibs)
2345   {
2346     if (outside_fib->fib_index == new_fib_index)
2347       {
2348         outside_fib->refcount++;
2349         is_add = 0;
2350         break;
2351       }
2352   }
2353
2354   if (is_add)
2355     {
2356       vec_add2 (sm->outside_fibs, outside_fib, 1);
2357       outside_fib->refcount = 1;
2358       outside_fib->fib_index = new_fib_index;
2359     }
2360 }
2361
2362 static void
2363 snat_ip4_table_bind (ip4_main_t * im,
2364                      uword opaque,
2365                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2366 {
2367   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2368 }
2369
2370 static void
2371 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2372                                        uword opaque,
2373                                        u32 sw_if_index,
2374                                        ip4_address_t * address,
2375                                        u32 address_length,
2376                                        u32 if_address_index, u32 is_delete);
2377
2378 static void
2379 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2380                                  uword opaque,
2381                                  u32 sw_if_index,
2382                                  ip4_address_t * address,
2383                                  u32 address_length,
2384                                  u32 if_address_index, u32 is_delete);
2385
2386 static int
2387 nat_alloc_addr_and_port_default (snat_address_t * addresses, u32 fib_index,
2388                                  u32 thread_index, nat_protocol_t proto,
2389                                  ip4_address_t * addr, u16 * port,
2390                                  u16 port_per_thread, u32 snat_thread_index);
2391
2392 void
2393 test_key_calc_split ()
2394 {
2395   ip4_address_t l_addr;
2396   l_addr.as_u8[0] = 1;
2397   l_addr.as_u8[1] = 1;
2398   l_addr.as_u8[2] = 1;
2399   l_addr.as_u8[3] = 1;
2400   ip4_address_t r_addr;
2401   r_addr.as_u8[0] = 2;
2402   r_addr.as_u8[1] = 2;
2403   r_addr.as_u8[2] = 2;
2404   r_addr.as_u8[3] = 2;
2405   u16 l_port = 40001;
2406   u16 r_port = 40301;
2407   u8 proto = 9;
2408   u32 fib_index = 9000001;
2409   u32 thread_index = 3000000001;
2410   u32 session_index = 3000000221;
2411   clib_bihash_kv_16_8_t kv;
2412   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2413               thread_index, session_index);
2414   ip4_address_t l_addr2;
2415   ip4_address_t r_addr2;
2416   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2417   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2418   u16 l_port2 = 0;
2419   u16 r_port2 = 0;
2420   u8 proto2 = 0;
2421   u32 fib_index2 = 0;
2422   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2423                &r_port2);
2424   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2425   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2426   ASSERT (l_port == l_port2);
2427   ASSERT (r_port == r_port2);
2428   ASSERT (proto == proto2);
2429   ASSERT (fib_index == fib_index2);
2430   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2431   ASSERT (session_index == ed_value_get_session_index (&kv));
2432
2433   fib_index = 7001;
2434   proto = 5;
2435   nat_protocol_t proto3 = ~0;
2436   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2437   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2438   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2439   ASSERT (l_port == l_port2);
2440   ASSERT (proto == proto3);
2441   ASSERT (fib_index == fib_index2);
2442 }
2443
2444 static clib_error_t *
2445 snat_init (vlib_main_t * vm)
2446 {
2447   snat_main_t *sm = &snat_main;
2448   clib_error_t *error = 0;
2449   ip4_main_t *im = &ip4_main;
2450   ip_lookup_main_t *lm = &im->lookup_main;
2451   uword *p;
2452   vlib_thread_registration_t *tr;
2453   vlib_thread_main_t *tm = vlib_get_thread_main ();
2454   uword *bitmap = 0;
2455   u32 i;
2456   ip4_add_del_interface_address_callback_t cb4;
2457   vlib_node_t *node;
2458
2459   sm->vnet_main = vnet_get_main ();
2460   sm->ip4_main = im;
2461   sm->ip4_lookup_main = lm;
2462   sm->api_main = vlibapi_get_main ();
2463   sm->first_worker_index = 0;
2464   sm->num_workers = 0;
2465   sm->workers = 0;
2466   sm->port_per_thread = 0xffff - 1024;
2467   sm->fq_in2out_index = ~0;
2468   sm->fq_in2out_output_index = ~0;
2469   sm->fq_out2in_index = ~0;
2470
2471   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2472   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2473   sm->forwarding_enabled = 0;
2474   sm->log_class = vlib_log_register_class ("nat", 0);
2475   sm->log_level = SNAT_LOG_ERROR;
2476   sm->mss_clamping = 0;
2477
2478   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2479   sm->error_node_index = node->index;
2480
2481   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2482   sm->pre_in2out_node_index = node->index;
2483   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2484   sm->pre_out2in_node_index = node->index;
2485
2486   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2487   sm->pre_in2out_node_index = node->index;
2488
2489   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2490   sm->pre_out2in_node_index = node->index;
2491
2492   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2493   sm->in2out_node_index = node->index;
2494   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2495   sm->in2out_output_node_index = node->index;
2496   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2497   sm->in2out_fast_node_index = node->index;
2498   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2499   sm->in2out_slowpath_node_index = node->index;
2500   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2501   sm->in2out_slowpath_output_node_index = node->index;
2502
2503   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2504   sm->ed_in2out_node_index = node->index;
2505   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2506   sm->ed_in2out_slowpath_node_index = node->index;
2507
2508   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2509   sm->out2in_node_index = node->index;
2510   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2511   sm->out2in_fast_node_index = node->index;
2512
2513   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2514   sm->ed_out2in_node_index = node->index;
2515   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2516   sm->ed_out2in_slowpath_node_index = node->index;
2517
2518   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2519   sm->det_in2out_node_index = node->index;
2520   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2521   sm->det_out2in_node_index = node->index;
2522
2523   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2524   sm->hairpinning_node_index = node->index;
2525   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2526   sm->hairpin_dst_node_index = node->index;
2527   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2528   sm->hairpin_src_node_index = node->index;
2529   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2530   sm->ed_hairpinning_node_index = node->index;
2531   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2532   sm->ed_hairpin_dst_node_index = node->index;
2533   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2534   sm->ed_hairpin_src_node_index = node->index;
2535
2536   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2537   if (p)
2538     {
2539       tr = (vlib_thread_registration_t *) p[0];
2540       if (tr)
2541         {
2542           sm->num_workers = tr->count;
2543           sm->first_worker_index = tr->first_index;
2544         }
2545     }
2546
2547   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2548
2549   /* Use all available workers by default */
2550   if (sm->num_workers > 1)
2551     {
2552       for (i = 0; i < sm->num_workers; i++)
2553         bitmap = clib_bitmap_set (bitmap, i, 1);
2554       snat_set_workers (bitmap);
2555       clib_bitmap_free (bitmap);
2556     }
2557   else
2558     {
2559       sm->per_thread_data[0].snat_thread_index = 0;
2560     }
2561
2562   error = snat_api_init (vm, sm);
2563   if (error)
2564     return error;
2565
2566   /* Set up the interface address add/del callback */
2567   cb4.function = snat_ip4_add_del_interface_address_cb;
2568   cb4.function_opaque = 0;
2569
2570   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2571
2572   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2573   cb4.function_opaque = 0;
2574
2575   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2576
2577   nat_dpo_module_init ();
2578
2579   /* Init counters */
2580   sm->total_users.name = "total-users";
2581   sm->total_users.stat_segment_name = "/nat44/total-users";
2582   vlib_validate_simple_counter (&sm->total_users, 0);
2583   vlib_zero_simple_counter (&sm->total_users, 0);
2584   sm->total_sessions.name = "total-sessions";
2585   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2586   vlib_validate_simple_counter (&sm->total_sessions, 0);
2587   vlib_zero_simple_counter (&sm->total_sessions, 0);
2588   sm->user_limit_reached.name = "user-limit-reached";
2589   sm->user_limit_reached.stat_segment_name = "/nat44/user-limit-reached";
2590   vlib_validate_simple_counter (&sm->user_limit_reached, 0);
2591   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2592
2593   /* Init IPFIX logging */
2594   snat_ipfix_logging_init (vm);
2595
2596   /* Init NAT64 */
2597   error = nat64_init (vm);
2598   if (error)
2599     return error;
2600
2601   nat66_init (vm);
2602
2603   ip4_table_bind_callback_t cbt4 = {
2604     .function = snat_ip4_table_bind,
2605   };
2606   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2607
2608   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2609                                         FIB_SOURCE_PRIORITY_HI,
2610                                         FIB_SOURCE_BH_SIMPLE);
2611   nat_fib_src_low = fib_source_allocate ("nat-low",
2612                                          FIB_SOURCE_PRIORITY_LOW,
2613                                          FIB_SOURCE_BH_SIMPLE);
2614
2615   test_key_calc_split ();
2616   return error;
2617 }
2618
2619 VLIB_INIT_FUNCTION (snat_init);
2620
2621 void
2622 snat_free_outside_address_and_port (snat_address_t * addresses,
2623                                     u32 thread_index,
2624                                     ip4_address_t * addr,
2625                                     u16 port, nat_protocol_t protocol)
2626 {
2627   snat_address_t *a;
2628   u32 address_index;
2629   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2630
2631   for (address_index = 0; address_index < vec_len (addresses);
2632        address_index++)
2633     {
2634       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2635         break;
2636     }
2637
2638   ASSERT (address_index < vec_len (addresses));
2639
2640   a = addresses + address_index;
2641
2642   switch (protocol)
2643     {
2644 #define _(N, i, n, s) \
2645     case NAT_PROTOCOL_##N: \
2646       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2647       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2648       a->busy_##n##_ports--; \
2649       a->busy_##n##_ports_per_thread[thread_index]--; \
2650       break;
2651       foreach_nat_protocol
2652 #undef _
2653     default:
2654       nat_elog_info ("unknown protocol");
2655       return;
2656     }
2657 }
2658
2659 static int
2660 nat_set_outside_address_and_port (snat_address_t * addresses,
2661                                   u32 thread_index, ip4_address_t addr,
2662                                   u16 port, nat_protocol_t protocol)
2663 {
2664   snat_address_t *a = 0;
2665   u32 address_index;
2666   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2667
2668   for (address_index = 0; address_index < vec_len (addresses);
2669        address_index++)
2670     {
2671       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2672         continue;
2673
2674       a = addresses + address_index;
2675       switch (protocol)
2676         {
2677 #define _(N, j, n, s) \
2678         case NAT_PROTOCOL_##N: \
2679           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2680             return VNET_API_ERROR_INSTANCE_IN_USE; \
2681           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2682           a->busy_##n##_ports_per_thread[thread_index]++; \
2683           a->busy_##n##_ports++; \
2684           return 0;
2685           foreach_nat_protocol
2686 #undef _
2687         default:
2688           nat_elog_info ("unknown protocol");
2689           return 1;
2690         }
2691     }
2692
2693   return VNET_API_ERROR_NO_SUCH_ENTRY;
2694 }
2695
2696 int
2697 snat_static_mapping_match (snat_main_t * sm,
2698                            ip4_address_t match_addr,
2699                            u16 match_port,
2700                            u32 match_fib_index,
2701                            nat_protocol_t match_protocol,
2702                            ip4_address_t * mapping_addr,
2703                            u16 * mapping_port,
2704                            u32 * mapping_fib_index,
2705                            u8 by_external,
2706                            u8 * is_addr_only,
2707                            twice_nat_type_t * twice_nat,
2708                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2709                            u8 * is_identity_nat)
2710 {
2711   clib_bihash_kv_8_8_t kv, value;
2712   snat_static_mapping_t *m;
2713   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2714   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2715   u8 backend_index;
2716   nat44_lb_addr_port_t *local;
2717
2718   if (by_external)
2719     {
2720       mapping_hash = &sm->static_mapping_by_external;
2721       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2722       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2723         {
2724           /* Try address only mapping */
2725           init_nat_k (&kv, match_addr, 0, 0, 0);
2726           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2727             return 1;
2728         }
2729
2730     }
2731   else
2732     {
2733       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2734                   match_protocol);
2735       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2736         {
2737           /* Try address only mapping */
2738           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2739           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2740             return 1;
2741         }
2742
2743     }
2744
2745   m = pool_elt_at_index (sm->static_mappings, value.value);
2746
2747   if (by_external)
2748     {
2749       if (is_lb_static_mapping (m))
2750         {
2751           if (PREDICT_FALSE (lb != 0))
2752             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2753           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2754                                                           match_addr,
2755                                                           match_protocol,
2756                                                           match_port,
2757                                                           &backend_index))
2758             {
2759               local = pool_elt_at_index (m->locals, backend_index);
2760               *mapping_addr = local->addr;
2761               *mapping_port = local->port;
2762               *mapping_fib_index = local->fib_index;
2763               goto end;
2764             }
2765           // pick locals matching this worker
2766           if (PREDICT_FALSE (sm->num_workers > 1))
2767             {
2768               u32 thread_index = vlib_get_thread_index ();
2769               /* *INDENT-OFF* */
2770               pool_foreach_index (i, m->locals,
2771               ({
2772                 local = pool_elt_at_index (m->locals, i);
2773
2774                 ip4_header_t ip = {
2775                   .src_address = local->addr,
2776                 };
2777
2778                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2779                     thread_index)
2780                   {
2781                     vec_add1 (tmp, i);
2782                   }
2783               }));
2784               /* *INDENT-ON* */
2785               ASSERT (vec_len (tmp) != 0);
2786             }
2787           else
2788             {
2789               /* *INDENT-OFF* */
2790               pool_foreach_index (i, m->locals,
2791               ({
2792                 vec_add1 (tmp, i);
2793               }));
2794               /* *INDENT-ON* */
2795             }
2796           hi = vec_len (tmp) - 1;
2797           local = pool_elt_at_index (m->locals, tmp[hi]);
2798           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2799           while (lo < hi)
2800             {
2801               mid = ((hi - lo) >> 1) + lo;
2802               local = pool_elt_at_index (m->locals, tmp[mid]);
2803               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2804             }
2805           local = pool_elt_at_index (m->locals, tmp[lo]);
2806           if (!(local->prefix >= rand))
2807             return 1;
2808           *mapping_addr = local->addr;
2809           *mapping_port = local->port;
2810           *mapping_fib_index = local->fib_index;
2811           if (m->affinity)
2812             {
2813               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2814                                                 match_protocol, match_port,
2815                                                 tmp[lo], m->affinity,
2816                                                 m->affinity_per_service_list_head_index))
2817                 nat_elog_info ("create affinity record failed");
2818             }
2819           vec_free (tmp);
2820         }
2821       else
2822         {
2823           if (PREDICT_FALSE (lb != 0))
2824             *lb = NO_LB_NAT;
2825           *mapping_fib_index = m->fib_index;
2826           *mapping_addr = m->local_addr;
2827           /* Address only mapping doesn't change port */
2828           *mapping_port = is_addr_only_static_mapping (m) ? match_port
2829             : m->local_port;
2830         }
2831     }
2832   else
2833     {
2834       *mapping_addr = m->external_addr;
2835       /* Address only mapping doesn't change port */
2836       *mapping_port = is_addr_only_static_mapping (m) ? match_port
2837         : m->external_port;
2838       *mapping_fib_index = sm->outside_fib_index;
2839     }
2840
2841 end:
2842   if (PREDICT_FALSE (is_addr_only != 0))
2843     *is_addr_only = is_addr_only_static_mapping (m);
2844
2845   if (PREDICT_FALSE (twice_nat != 0))
2846     *twice_nat = m->twice_nat;
2847
2848   if (PREDICT_FALSE (is_identity_nat != 0))
2849     *is_identity_nat = is_identity_static_mapping (m);
2850
2851   return 0;
2852 }
2853
2854 static_always_inline u16
2855 snat_random_port (u16 min, u16 max)
2856 {
2857   snat_main_t *sm = &snat_main;
2858   return min + random_u32 (&sm->random_seed) /
2859     (random_u32_max () / (max - min + 1) + 1);
2860 }
2861
2862 int
2863 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2864                                      u32 fib_index,
2865                                      u32 thread_index,
2866                                      nat_protocol_t proto,
2867                                      ip4_address_t * addr,
2868                                      u16 * port,
2869                                      u16 port_per_thread,
2870                                      u32 snat_thread_index)
2871 {
2872   snat_main_t *sm = &snat_main;
2873
2874   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, proto,
2875                                   addr, port, port_per_thread,
2876                                   snat_thread_index);
2877 }
2878
2879 static int
2880 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2881                                  u32 fib_index,
2882                                  u32 thread_index,
2883                                  nat_protocol_t proto,
2884                                  ip4_address_t * addr,
2885                                  u16 * port,
2886                                  u16 port_per_thread, u32 snat_thread_index)
2887 {
2888   int i;
2889   snat_address_t *a, *ga = 0;
2890   u32 portnum;
2891
2892   for (i = 0; i < vec_len (addresses); i++)
2893     {
2894       a = addresses + i;
2895       switch (proto)
2896         {
2897 #define _(N, j, n, s) \
2898         case NAT_PROTOCOL_##N: \
2899           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2900             { \
2901               if (a->fib_index == fib_index) \
2902                 { \
2903                   while (1) \
2904                     { \
2905                       portnum = (port_per_thread * \
2906                         snat_thread_index) + \
2907                         snat_random_port(1, port_per_thread) + 1024; \
2908                       if (a->busy_##n##_port_refcounts[portnum]) \
2909                         continue; \
2910                       --a->busy_##n##_port_refcounts[portnum]; \
2911                       a->busy_##n##_ports_per_thread[thread_index]++; \
2912                       a->busy_##n##_ports++; \
2913                       *addr = a->addr; \
2914                       *port = clib_host_to_net_u16(portnum); \
2915                       return 0; \
2916                     } \
2917                 } \
2918               else if (a->fib_index == ~0) \
2919                 { \
2920                   ga = a; \
2921                 } \
2922             } \
2923           break;
2924           foreach_nat_protocol
2925 #undef _
2926         default:
2927           nat_elog_info ("unknown protocol");
2928           return 1;
2929         }
2930
2931     }
2932
2933   if (ga)
2934     {
2935       a = ga;
2936       switch (proto)
2937         {
2938 #define _(N, j, n, s) \
2939         case NAT_PROTOCOL_##N: \
2940           while (1) \
2941             { \
2942               portnum = (port_per_thread * \
2943                 snat_thread_index) + \
2944                 snat_random_port(1, port_per_thread) + 1024; \
2945               if (a->busy_##n##_port_refcounts[portnum]) \
2946                 continue; \
2947               ++a->busy_##n##_port_refcounts[portnum]; \
2948               a->busy_##n##_ports_per_thread[thread_index]++; \
2949               a->busy_##n##_ports++; \
2950               *addr = a->addr; \
2951               *port = clib_host_to_net_u16(portnum); \
2952               return 0; \
2953             }
2954           break;
2955           foreach_nat_protocol
2956 #undef _
2957         default:
2958           nat_elog_info ("unknown protocol");
2959           return 1;
2960         }
2961     }
2962
2963   /* Totally out of translations to use... */
2964   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2965   return 1;
2966 }
2967
2968 static int
2969 nat_alloc_addr_and_port_mape (snat_address_t * addresses, u32 fib_index,
2970                               u32 thread_index, nat_protocol_t proto,
2971                               ip4_address_t * addr, u16 * port,
2972                               u16 port_per_thread, u32 snat_thread_index)
2973 {
2974   snat_main_t *sm = &snat_main;
2975   snat_address_t *a = addresses;
2976   u16 m, ports, portnum, A, j;
2977   m = 16 - (sm->psid_offset + sm->psid_length);
2978   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2979
2980   if (!vec_len (addresses))
2981     goto exhausted;
2982
2983   switch (proto)
2984     {
2985 #define _(N, i, n, s) \
2986     case NAT_PROTOCOL_##N: \
2987       if (a->busy_##n##_ports < ports) \
2988         { \
2989           while (1) \
2990             { \
2991               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2992               j = snat_random_port(0, pow2_mask(m)); \
2993               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2994               if (a->busy_##n##_port_refcounts[portnum]) \
2995                 continue; \
2996               ++a->busy_##n##_port_refcounts[portnum]; \
2997               a->busy_##n##_ports++; \
2998               *addr = a->addr; \
2999               *port = clib_host_to_net_u16 (portnum); \
3000               return 0; \
3001             } \
3002         } \
3003       break;
3004       foreach_nat_protocol
3005 #undef _
3006     default:
3007       nat_elog_info ("unknown protocol");
3008       return 1;
3009     }
3010
3011 exhausted:
3012   /* Totally out of translations to use... */
3013   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3014   return 1;
3015 }
3016
3017 static int
3018 nat_alloc_addr_and_port_range (snat_address_t * addresses, u32 fib_index,
3019                                u32 thread_index, nat_protocol_t proto,
3020                                ip4_address_t * addr, u16 * port,
3021                                u16 port_per_thread, u32 snat_thread_index)
3022 {
3023   snat_main_t *sm = &snat_main;
3024   snat_address_t *a = addresses;
3025   u16 portnum, ports;
3026
3027   ports = sm->end_port - sm->start_port + 1;
3028
3029   if (!vec_len (addresses))
3030     goto exhausted;
3031
3032   switch (proto)
3033     {
3034 #define _(N, i, n, s) \
3035     case NAT_PROTOCOL_##N: \
3036       if (a->busy_##n##_ports < ports) \
3037         { \
3038           while (1) \
3039             { \
3040               portnum = snat_random_port(sm->start_port, sm->end_port); \
3041               if (a->busy_##n##_port_refcounts[portnum]) \
3042                 continue; \
3043               ++a->busy_##n##_port_refcounts[portnum]; \
3044               a->busy_##n##_ports++; \
3045               *addr = a->addr; \
3046               *port = clib_host_to_net_u16 (portnum); \
3047               return 0; \
3048             } \
3049         } \
3050       break;
3051       foreach_nat_protocol
3052 #undef _
3053     default:
3054       nat_elog_info ("unknown protocol");
3055       return 1;
3056     }
3057
3058 exhausted:
3059   /* Totally out of translations to use... */
3060   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3061   return 1;
3062 }
3063
3064 void
3065 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3066 {
3067   dpo_id_t dpo_v4 = DPO_INVALID;
3068   fib_prefix_t pfx = {
3069     .fp_proto = FIB_PROTOCOL_IP4,
3070     .fp_len = 32,
3071     .fp_addr.ip4.as_u32 = addr.as_u32,
3072   };
3073
3074   if (is_add)
3075     {
3076       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3077       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3078                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3079       dpo_reset (&dpo_v4);
3080     }
3081   else
3082     {
3083       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3084     }
3085 }
3086
3087 u8 *
3088 format_session_kvp (u8 * s, va_list * args)
3089 {
3090   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3091
3092   s = format (s, "%U session-index %llu", format_snat_key, v->key, v->value);
3093
3094   return s;
3095 }
3096
3097 u8 *
3098 format_static_mapping_kvp (u8 * s, va_list * args)
3099 {
3100   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3101
3102   s = format (s, "%U static-mapping-index %llu",
3103               format_snat_key, v->key, v->value);
3104
3105   return s;
3106 }
3107
3108 u8 *
3109 format_user_kvp (u8 * s, va_list * args)
3110 {
3111   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3112   snat_user_key_t k;
3113
3114   k.as_u64 = v->key;
3115
3116   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3117               k.fib_index, v->value);
3118
3119   return s;
3120 }
3121
3122 u8 *
3123 format_ed_session_kvp (u8 * s, va_list * args)
3124 {
3125   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3126
3127   u8 proto;
3128   u16 r_port, l_port;
3129   ip4_address_t l_addr, r_addr;
3130   u32 fib_index;
3131
3132   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
3133   s =
3134     format (s,
3135             "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
3136             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
3137             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
3138             format_ip_protocol, proto, fib_index,
3139             ed_value_get_session_index (v), ed_value_get_thread_index (v));
3140
3141   return s;
3142 }
3143
3144 static u32
3145 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3146                            u8 is_output)
3147 {
3148   snat_main_t *sm = &snat_main;
3149   u32 next_worker_index = 0;
3150   u32 hash;
3151
3152   next_worker_index = sm->first_worker_index;
3153   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3154     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3155
3156   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3157     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3158   else
3159     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3160
3161   return next_worker_index;
3162 }
3163
3164 static u32
3165 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3166                            u32 rx_fib_index0, u8 is_output)
3167 {
3168   snat_main_t *sm = &snat_main;
3169   udp_header_t *udp;
3170   u16 port;
3171   clib_bihash_kv_8_8_t kv, value;
3172   snat_static_mapping_t *m;
3173   u32 proto;
3174   u32 next_worker_index = 0;
3175
3176   /* first try static mappings without port */
3177   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3178     {
3179       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
3180       if (!clib_bihash_search_8_8
3181           (&sm->static_mapping_by_external, &kv, &value))
3182         {
3183           m = pool_elt_at_index (sm->static_mappings, value.value);
3184           return m->workers[0];
3185         }
3186     }
3187
3188   proto = ip_proto_to_nat_proto (ip0->protocol);
3189   udp = ip4_next_header (ip0);
3190   port = udp->dst_port;
3191
3192   /* unknown protocol */
3193   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3194     {
3195       /* use current thread */
3196       return vlib_get_thread_index ();
3197     }
3198
3199   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3200     {
3201       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3202       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3203       if (!icmp_type_is_error_message
3204           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3205         port = vnet_buffer (b)->ip.reass.l4_src_port;
3206       else
3207         {
3208           /* if error message, then it's not fragmented and we can access it */
3209           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3210           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3211           void *l4_header = ip4_next_header (inner_ip);
3212           switch (proto)
3213             {
3214             case NAT_PROTOCOL_ICMP:
3215               icmp = (icmp46_header_t *) l4_header;
3216               echo = (icmp_echo_header_t *) (icmp + 1);
3217               port = echo->identifier;
3218               break;
3219             case NAT_PROTOCOL_UDP:
3220             case NAT_PROTOCOL_TCP:
3221               port = ((tcp_udp_header_t *) l4_header)->src_port;
3222               break;
3223             default:
3224               return vlib_get_thread_index ();
3225             }
3226         }
3227     }
3228
3229   /* try static mappings with port */
3230   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3231     {
3232       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
3233       if (!clib_bihash_search_8_8
3234           (&sm->static_mapping_by_external, &kv, &value))
3235         {
3236           m = pool_elt_at_index (sm->static_mappings, value.value);
3237           return m->workers[0];
3238         }
3239     }
3240
3241   /* worker by outside port */
3242   next_worker_index = sm->first_worker_index;
3243   next_worker_index +=
3244     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3245   return next_worker_index;
3246 }
3247
3248 static u32
3249 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3250                                u8 is_output)
3251 {
3252   snat_main_t *sm = &snat_main;
3253   u32 next_worker_index = sm->first_worker_index;
3254   u32 hash;
3255
3256   clib_bihash_kv_16_8_t kv16, value16;
3257   snat_main_per_thread_data_t *tsm;
3258   udp_header_t *udp;
3259
3260   if (PREDICT_FALSE (is_output))
3261     {
3262       u32 fib_index = sm->outside_fib_index;
3263       nat_outside_fib_t *outside_fib;
3264       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3265       fib_prefix_t pfx = {
3266         .fp_proto = FIB_PROTOCOL_IP4,
3267         .fp_len = 32,
3268         .fp_addr = {
3269                     .ip4.as_u32 = ip->dst_address.as_u32,
3270                     }
3271         ,
3272       };
3273
3274       udp = ip4_next_header (ip);
3275
3276       switch (vec_len (sm->outside_fibs))
3277         {
3278         case 0:
3279           fib_index = sm->outside_fib_index;
3280           break;
3281         case 1:
3282           fib_index = sm->outside_fibs[0].fib_index;
3283           break;
3284         default:
3285             /* *INDENT-OFF* */
3286             vec_foreach (outside_fib, sm->outside_fibs)
3287               {
3288                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3289                 if (FIB_NODE_INDEX_INVALID != fei)
3290                   {
3291                     if (fib_entry_get_resolving_interface (fei) != ~0)
3292                       {
3293                         fib_index = outside_fib->fib_index;
3294                         break;
3295                       }
3296                   }
3297               }
3298             /* *INDENT-ON* */
3299           break;
3300         }
3301
3302       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3303                  udp->dst_port, fib_index, ip->protocol);
3304
3305       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3306                                                   &kv16, &value16)))
3307         {
3308           tsm =
3309             vec_elt_at_index (sm->per_thread_data,
3310                               ed_value_get_thread_index (&value16));
3311           next_worker_index += tsm->thread_index;
3312
3313           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3314                                   next_worker_index, fib_index,
3315                                   clib_net_to_host_u32 (ip->
3316                                                         src_address.as_u32),
3317                                   clib_net_to_host_u32 (ip->
3318                                                         dst_address.as_u32));
3319
3320           return next_worker_index;
3321         }
3322     }
3323
3324   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3325     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3326
3327   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3328     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3329   else
3330     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3331
3332   if (PREDICT_TRUE (!is_output))
3333     {
3334       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3335                               next_worker_index, rx_fib_index,
3336                               clib_net_to_host_u32 (ip->src_address.as_u32),
3337                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3338     }
3339   else
3340     {
3341       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3342                               next_worker_index, rx_fib_index,
3343                               clib_net_to_host_u32 (ip->src_address.as_u32),
3344                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3345     }
3346
3347   return next_worker_index;
3348 }
3349
3350 static u32
3351 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3352                                u32 rx_fib_index, u8 is_output)
3353 {
3354   snat_main_t *sm = &snat_main;
3355   clib_bihash_kv_8_8_t kv, value;
3356   clib_bihash_kv_16_8_t kv16, value16;
3357   snat_main_per_thread_data_t *tsm;
3358
3359   u32 proto, next_worker_index = 0;
3360   udp_header_t *udp;
3361   u16 port;
3362   snat_static_mapping_t *m;
3363   u32 hash;
3364
3365   proto = ip_proto_to_nat_proto (ip->protocol);
3366
3367   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3368     {
3369       udp = ip4_next_header (ip);
3370
3371       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3372                  udp->src_port, rx_fib_index, ip->protocol);
3373
3374       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3375                                                   &kv16, &value16)))
3376         {
3377           tsm =
3378             vec_elt_at_index (sm->per_thread_data,
3379                               ed_value_get_thread_index (&value16));
3380           vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
3381             ed_value_get_session_index (&value16);
3382           next_worker_index = sm->first_worker_index + tsm->thread_index;
3383           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3384                                   next_worker_index, rx_fib_index,
3385                                   clib_net_to_host_u32 (ip->
3386                                                         src_address.as_u32),
3387                                   clib_net_to_host_u32 (ip->
3388                                                         dst_address.as_u32));
3389           return next_worker_index;
3390         }
3391     }
3392   else if (proto == NAT_PROTOCOL_ICMP)
3393     {
3394       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
3395         {
3396           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3397                                                       &kv16, &value16)))
3398             {
3399               tsm =
3400                 vec_elt_at_index (sm->per_thread_data,
3401                                   ed_value_get_thread_index (&value16));
3402               next_worker_index = sm->first_worker_index + tsm->thread_index;
3403               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3404                                       next_worker_index, rx_fib_index,
3405                                       clib_net_to_host_u32 (ip->
3406                                                             src_address.as_u32),
3407                                       clib_net_to_host_u32 (ip->
3408                                                             dst_address.as_u32));
3409               return next_worker_index;
3410             }
3411         }
3412     }
3413
3414   /* first try static mappings without port */
3415   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3416     {
3417       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3418       if (!clib_bihash_search_8_8
3419           (&sm->static_mapping_by_external, &kv, &value))
3420         {
3421           m = pool_elt_at_index (sm->static_mappings, value.value);
3422           next_worker_index = m->workers[0];
3423           goto done;
3424         }
3425     }
3426
3427   /* unknown protocol */
3428   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3429     {
3430       /* use current thread */
3431       next_worker_index = vlib_get_thread_index ();
3432       goto done;
3433     }
3434
3435   udp = ip4_next_header (ip);
3436   port = udp->dst_port;
3437
3438   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3439     {
3440       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3441       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3442       if (!icmp_type_is_error_message
3443           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3444         port = vnet_buffer (b)->ip.reass.l4_src_port;
3445       else
3446         {
3447           /* if error message, then it's not fragmented and we can access it */
3448           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3449           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3450           void *l4_header = ip4_next_header (inner_ip);
3451           switch (proto)
3452             {
3453             case NAT_PROTOCOL_ICMP:
3454               icmp = (icmp46_header_t *) l4_header;
3455               echo = (icmp_echo_header_t *) (icmp + 1);
3456               port = echo->identifier;
3457               break;
3458             case NAT_PROTOCOL_UDP:
3459             case NAT_PROTOCOL_TCP:
3460               port = ((tcp_udp_header_t *) l4_header)->src_port;
3461               break;
3462             default:
3463               next_worker_index = vlib_get_thread_index ();
3464               goto done;
3465             }
3466         }
3467     }
3468
3469   /* try static mappings with port */
3470   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3471     {
3472       init_nat_k (&kv, ip->dst_address, proto, 0, port);
3473       if (!clib_bihash_search_8_8
3474           (&sm->static_mapping_by_external, &kv, &value))
3475         {
3476           m = pool_elt_at_index (sm->static_mappings, value.value);
3477           if (!is_lb_static_mapping (m))
3478             {
3479               next_worker_index = m->workers[0];
3480               goto done;
3481             }
3482
3483           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3484             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3485
3486           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3487             next_worker_index =
3488               m->workers[hash & (_vec_len (m->workers) - 1)];
3489           else
3490             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3491           goto done;
3492         }
3493     }
3494
3495   /* worker by outside port */
3496   next_worker_index = sm->first_worker_index;
3497   next_worker_index +=
3498     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3499
3500 done:
3501   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3502                           clib_net_to_host_u32 (ip->src_address.as_u32),
3503                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3504   return next_worker_index;
3505 }
3506
3507 void
3508 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3509                 ip4_address_t * out_addr, u16 out_port,
3510                 ip4_address_t * eh_addr, u16 eh_port,
3511                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3512                 u32 fib_index, u16 flags, u32 thread_index)
3513 {
3514   snat_main_t *sm = &snat_main;
3515   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3516   snat_user_t *u;
3517   snat_session_t *s;
3518   clib_bihash_kv_8_8_t kv;
3519   vlib_main_t *vm = vlib_get_main ();
3520   f64 now = vlib_time_now (vm);
3521   nat_outside_fib_t *outside_fib;
3522   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3523   fib_prefix_t pfx = {
3524     .fp_proto = FIB_PROTOCOL_IP4,
3525     .fp_len = 32,
3526     .fp_addr = {
3527                 .ip4.as_u32 = eh_addr->as_u32,
3528                 },
3529   };
3530
3531   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3532     {
3533       if (nat_set_outside_address_and_port
3534           (sm->addresses, thread_index, *out_addr, out_port, proto))
3535         return;
3536     }
3537
3538   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3539   if (!u)
3540     return;
3541
3542   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3543   if (!s)
3544     return;
3545
3546   if (sm->endpoint_dependent)
3547     {
3548       nat_ed_lru_insert (tsm, s, now, nat_proto_to_ip_proto (proto));
3549     }
3550
3551   s->out2in.addr.as_u32 = out_addr->as_u32;
3552   s->out2in.port = out_port;
3553   s->nat_proto = proto;
3554   s->last_heard = now;
3555   s->flags = flags;
3556   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3557   s->ext_host_port = eh_port;
3558   user_session_increment (sm, u, snat_is_session_static (s));
3559   switch (vec_len (sm->outside_fibs))
3560     {
3561     case 0:
3562       s->out2in.fib_index = sm->outside_fib_index;
3563       break;
3564     case 1:
3565       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3566       break;
3567     default:
3568       /* *INDENT-OFF* */
3569       vec_foreach (outside_fib, sm->outside_fibs)
3570         {
3571           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3572           if (FIB_NODE_INDEX_INVALID != fei)
3573             {
3574               if (fib_entry_get_resolving_interface (fei) != ~0)
3575                 {
3576                   s->out2in.fib_index = outside_fib->fib_index;
3577                   break;
3578                 }
3579             }
3580         }
3581       /* *INDENT-ON* */
3582       break;
3583     }
3584   init_nat_o2i_kv (&kv, s, s - tsm->sessions);
3585   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3586     nat_elog_warn ("out2in key add failed");
3587
3588   s->in2out.addr.as_u32 = in_addr->as_u32;
3589   s->in2out.port = in_port;
3590   s->in2out.fib_index = fib_index;
3591   init_nat_i2o_kv (&kv, s, s - tsm->sessions);
3592   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3593     nat_elog_warn ("in2out key add failed");
3594 }
3595
3596 void
3597 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3598                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3599                 u32 ti)
3600 {
3601   snat_main_t *sm = &snat_main;
3602   clib_bihash_kv_8_8_t kv, value;
3603   u32 thread_index;
3604   snat_session_t *s;
3605   snat_main_per_thread_data_t *tsm;
3606
3607   if (sm->num_workers > 1)
3608     thread_index =
3609       sm->first_worker_index +
3610       (sm->workers[(clib_net_to_host_u16 (out_port) -
3611                     1024) / sm->port_per_thread]);
3612   else
3613     thread_index = sm->num_workers;
3614   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3615
3616   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3617   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3618     return;
3619
3620   s = pool_elt_at_index (tsm->sessions, value.value);
3621   nat_free_session_data (sm, s, thread_index, 1);
3622   nat44_delete_session (sm, s, thread_index);
3623 }
3624
3625 void
3626 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3627                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3628                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3629 {
3630   snat_main_t *sm = &snat_main;
3631   clib_bihash_kv_8_8_t kv, value;
3632   snat_session_t *s;
3633   snat_main_per_thread_data_t *tsm;
3634
3635   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3636
3637   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3638   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3639     return;
3640
3641   s = pool_elt_at_index (tsm->sessions, value.value);
3642   s->total_pkts = total_pkts;
3643   s->total_bytes = total_bytes;
3644 }
3645
3646 void
3647 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3648                    ip4_address_t * out_addr, u16 out_port,
3649                    ip4_address_t * eh_addr, u16 eh_port,
3650                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3651                    u32 fib_index, u16 flags, u32 thread_index)
3652 {
3653   snat_main_t *sm = &snat_main;
3654   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3655   snat_session_t *s;
3656   clib_bihash_kv_16_8_t kv;
3657   vlib_main_t *vm = vlib_get_main ();
3658   f64 now = vlib_time_now (vm);
3659   nat_outside_fib_t *outside_fib;
3660   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3661   fib_prefix_t pfx = {
3662     .fp_proto = FIB_PROTOCOL_IP4,
3663     .fp_len = 32,
3664     .fp_addr = {
3665                 .ip4.as_u32 = eh_addr->as_u32,
3666                 },
3667   };
3668
3669
3670   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3671     {
3672       if (nat_set_outside_address_and_port
3673           (sm->addresses, thread_index, *out_addr, out_port, proto))
3674         return;
3675     }
3676
3677   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3678     {
3679       if (nat_set_outside_address_and_port
3680           (sm->addresses, thread_index, *ehn_addr, ehn_port, proto))
3681         return;
3682     }
3683
3684   s = nat_ed_session_alloc (sm, thread_index, now, proto);
3685   if (!s)
3686     return;
3687
3688   s->last_heard = now;
3689   s->flags = flags;
3690   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3691   s->ext_host_nat_port = s->ext_host_port = eh_port;
3692   if (is_twice_nat_session (s))
3693     {
3694       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3695       s->ext_host_nat_port = ehn_port;
3696     }
3697   switch (vec_len (sm->outside_fibs))
3698     {
3699     case 0:
3700       s->out2in.fib_index = sm->outside_fib_index;
3701       break;
3702     case 1:
3703       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3704       break;
3705     default:
3706       /* *INDENT-OFF* */
3707       vec_foreach (outside_fib, sm->outside_fibs)
3708         {
3709           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3710           if (FIB_NODE_INDEX_INVALID != fei)
3711             {
3712               if (fib_entry_get_resolving_interface (fei) != ~0)
3713                 {
3714                   s->out2in.fib_index = outside_fib->fib_index;
3715                   break;
3716                 }
3717             }
3718         }
3719       /* *INDENT-ON* */
3720       break;
3721     }
3722   s->nat_proto = proto;
3723   s->out2in.addr.as_u32 = out_addr->as_u32;
3724   s->out2in.port = out_port;
3725
3726   s->in2out.addr.as_u32 = in_addr->as_u32;
3727   s->in2out.port = in_port;
3728   s->in2out.fib_index = fib_index;
3729
3730   init_ed_kv (&kv, *in_addr, in_port, s->ext_host_nat_addr,
3731               s->ext_host_nat_port, fib_index, nat_proto_to_ip_proto (proto),
3732               thread_index, s - tsm->sessions);
3733   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3734     nat_elog_warn ("in2out key add failed");
3735
3736   init_ed_kv (&kv, *out_addr, out_port, *eh_addr, eh_port,
3737               s->out2in.fib_index, nat_proto_to_ip_proto (proto),
3738               thread_index, s - tsm->sessions);
3739   if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 1))
3740     nat_elog_warn ("out2in key add failed");
3741 }
3742
3743 void
3744 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3745                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3746                    u32 fib_index, u32 ti)
3747 {
3748   snat_main_t *sm = &snat_main;
3749   clib_bihash_kv_16_8_t kv, value;
3750   u32 thread_index;
3751   snat_session_t *s;
3752   snat_main_per_thread_data_t *tsm;
3753
3754   if (sm->num_workers > 1)
3755     thread_index =
3756       sm->first_worker_index +
3757       (sm->workers[(clib_net_to_host_u16 (out_port) -
3758                     1024) / sm->port_per_thread]);
3759   else
3760     thread_index = sm->num_workers;
3761   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3762
3763   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3764   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3765     return;
3766
3767   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3768   nat_free_session_data (sm, s, thread_index, 1);
3769   nat44_delete_session (sm, s, thread_index);
3770 }
3771
3772 void
3773 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3774                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3775                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3776                    u32 thread_index)
3777 {
3778   snat_main_t *sm = &snat_main;
3779   clib_bihash_kv_16_8_t kv, value;
3780   snat_session_t *s;
3781   snat_main_per_thread_data_t *tsm;
3782
3783   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3784
3785   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3786   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3787     return;
3788
3789   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3790   s->total_pkts = total_pkts;
3791   s->total_bytes = total_bytes;
3792 }
3793
3794 static u32
3795 nat_calc_bihash_buckets (u32 n_elts)
3796 {
3797   return 1 << (max_log2 (n_elts >> 1) + 1);
3798 }
3799
3800 static u32
3801 nat_calc_bihash_memory (u32 n_buckets, uword kv_size)
3802 {
3803   return n_buckets * (8 + kv_size * 4);
3804 }
3805
3806 void
3807 nat44_db_init (snat_main_per_thread_data_t * tsm)
3808 {
3809   snat_main_t *sm = &snat_main;
3810
3811   pool_alloc (tsm->sessions, sm->max_translations_per_thread);
3812   pool_alloc (tsm->lru_pool, sm->max_translations_per_thread);
3813
3814   dlist_elt_t *head;
3815
3816   pool_get (tsm->lru_pool, head);
3817   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3818   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3819
3820   pool_get (tsm->lru_pool, head);
3821   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3822   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3823
3824   pool_get (tsm->lru_pool, head);
3825   tsm->udp_lru_head_index = head - tsm->lru_pool;
3826   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3827
3828   pool_get (tsm->lru_pool, head);
3829   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3830   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3831
3832   pool_get (tsm->lru_pool, head);
3833   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3834   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3835
3836   if (sm->endpoint_dependent)
3837     {
3838       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3839                              sm->translation_buckets,
3840                              sm->translation_memory_size);
3841       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3842                                           format_ed_session_kvp);
3843     }
3844   else
3845     {
3846       clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3847                             sm->translation_buckets,
3848                             sm->translation_memory_size);
3849       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
3850       clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3851                             sm->translation_buckets,
3852                             sm->translation_memory_size);
3853       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
3854     }
3855
3856   // TODO: resolve static mappings (put only to !ED)
3857   pool_alloc (tsm->list_pool, sm->max_translations_per_thread);
3858   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets,
3859                         sm->user_memory_size);
3860   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3861 }
3862
3863 void
3864 nat44_db_free (snat_main_per_thread_data_t * tsm)
3865 {
3866   snat_main_t *sm = &snat_main;
3867
3868   pool_free (tsm->sessions);
3869   pool_free (tsm->lru_pool);
3870
3871   if (sm->endpoint_dependent)
3872     {
3873       clib_bihash_free_16_8 (&tsm->in2out_ed);
3874     }
3875   else
3876     {
3877       clib_bihash_free_8_8 (&tsm->in2out);
3878       clib_bihash_free_8_8 (&tsm->out2in);
3879     }
3880
3881   // TODO: resolve static mappings (put only to !ED)
3882   pool_free (tsm->users);
3883   pool_free (tsm->list_pool);
3884   clib_bihash_free_8_8 (&tsm->user_hash);
3885 }
3886
3887 void
3888 nat44_sessions_clear ()
3889 {
3890   snat_main_t *sm = &snat_main;
3891   snat_main_per_thread_data_t *tsm;
3892
3893   if (sm->endpoint_dependent)
3894     {
3895       clib_bihash_free_16_8 (&sm->out2in_ed);
3896       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
3897                              clib_max (1, sm->num_workers) *
3898                              sm->translation_buckets,
3899                              clib_max (1, sm->num_workers) *
3900                              sm->translation_memory_size);
3901       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
3902                                           format_ed_session_kvp);
3903     }
3904
3905   /* *INDENT-OFF* */
3906   vec_foreach (tsm, sm->per_thread_data)
3907     {
3908       u32 ti;
3909
3910       nat44_db_free (tsm);
3911       nat44_db_init (tsm);
3912
3913       ti = tsm->snat_thread_index;
3914       vlib_set_simple_counter (&sm->total_users, ti, 0, 0);
3915       vlib_set_simple_counter (&sm->total_sessions, ti, 0, 0);
3916     }
3917   /* *INDENT-ON* */
3918 }
3919
3920 static clib_error_t *
3921 snat_config (vlib_main_t * vm, unformat_input_t * input)
3922 {
3923   snat_main_t *sm = &snat_main;
3924   nat66_main_t *nm = &nat66_main;
3925   snat_main_per_thread_data_t *tsm;
3926
3927   u32 static_mapping_buckets = 1024;
3928   uword static_mapping_memory_size = 64 << 20;
3929
3930   u32 nat64_bib_buckets = 1024;
3931   u32 nat64_bib_memory_size = 128 << 20;
3932
3933   u32 nat64_st_buckets = 2048;
3934   uword nat64_st_memory_size = 256 << 20;
3935
3936   u32 max_users_per_thread = 0;
3937   u32 user_memory_size = 0;
3938   u32 max_translations_per_thread = 0;
3939   u32 translation_memory_size = 0;
3940
3941   u32 max_translations_per_user = ~0;
3942
3943   u32 outside_vrf_id = 0;
3944   u32 outside_ip6_vrf_id = 0;
3945   u32 inside_vrf_id = 0;
3946   u8 static_mapping_only = 0;
3947   u8 static_mapping_connection_tracking = 0;
3948
3949   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3950   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3951   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3952   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3953
3954   sm->deterministic = 0;
3955   sm->out2in_dpo = 0;
3956   sm->endpoint_dependent = 0;
3957
3958   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3959     {
3960       if (unformat
3961           (input, "max translations per thread %d",
3962            &max_translations_per_thread))
3963         ;
3964       else if (unformat (input, "udp timeout %d", &udp_timeout))
3965         ;
3966       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3967         ;
3968       else if (unformat (input, "tcp transitory timeout %d",
3969                          &tcp_transitory_timeout));
3970       else if (unformat (input, "tcp established timeout %d",
3971                          &tcp_established_timeout));
3972       else if (unformat (input, "translation hash memory %d",
3973                          &translation_memory_size));
3974       else
3975         if (unformat
3976             (input, "max users per thread %d", &max_users_per_thread))
3977         ;
3978       else if (unformat (input, "user hash memory %d", &user_memory_size))
3979         ;
3980       else if (unformat (input, "max translations per user %d",
3981                          &max_translations_per_user))
3982         ;
3983       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3984         ;
3985       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3986         ;
3987       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3988         ;
3989       else if (unformat (input, "static mapping only"))
3990         {
3991           static_mapping_only = 1;
3992           if (unformat (input, "connection tracking"))
3993             static_mapping_connection_tracking = 1;
3994         }
3995       else if (unformat (input, "deterministic"))
3996         sm->deterministic = 1;
3997       else if (unformat (input, "nat64 bib hash buckets %d",
3998                          &nat64_bib_buckets))
3999         ;
4000       else if (unformat (input, "nat64 bib hash memory %d",
4001                          &nat64_bib_memory_size))
4002         ;
4003       else
4004         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
4005         ;
4006       else if (unformat (input, "nat64 st hash memory %d",
4007                          &nat64_st_memory_size))
4008         ;
4009       else if (unformat (input, "out2in dpo"))
4010         sm->out2in_dpo = 1;
4011       else if (unformat (input, "endpoint-dependent"))
4012         sm->endpoint_dependent = 1;
4013       else
4014         return clib_error_return (0, "unknown input '%U'",
4015                                   format_unformat_error, input);
4016     }
4017
4018   if (sm->deterministic && sm->endpoint_dependent)
4019     return clib_error_return (0,
4020                               "deterministic and endpoint-dependent modes are mutually exclusive");
4021
4022   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
4023     return clib_error_return (0,
4024                               "static mapping only mode available only for simple nat");
4025
4026   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
4027     return clib_error_return (0,
4028                               "out2in dpo mode available only for simple nat");
4029   if (sm->endpoint_dependent && max_users_per_thread > 0)
4030     {
4031       return clib_error_return (0,
4032                                 "setting 'max users' in endpoint-dependent mode is not supported");
4033     }
4034
4035   if (sm->endpoint_dependent && max_translations_per_user != ~0)
4036     {
4037       return clib_error_return (0,
4038                                 "setting 'max translations per user' in endpoint-dependent mode is not supported");
4039     }
4040
4041   /* optionally configurable timeouts for testing purposes */
4042   sm->udp_timeout = udp_timeout;
4043   sm->tcp_transitory_timeout = tcp_transitory_timeout;
4044   sm->tcp_established_timeout = tcp_established_timeout;
4045   sm->icmp_timeout = icmp_timeout;
4046
4047   if (0 == max_users_per_thread)
4048     {
4049       max_users_per_thread = 1024;
4050     }
4051   sm->max_users_per_thread = max_users_per_thread;
4052   sm->user_buckets = nat_calc_bihash_buckets (sm->max_users_per_thread);
4053
4054   if (0 == max_translations_per_thread)
4055     {
4056       // default value based on legacy setting of load factor 10 * default
4057       // translation buckets 1024
4058       max_translations_per_thread = 10 * 1024;
4059     }
4060   sm->max_translations_per_thread = max_translations_per_thread;
4061   sm->translation_buckets =
4062     nat_calc_bihash_buckets (sm->max_translations_per_thread);
4063   if (0 == translation_memory_size)
4064     {
4065       translation_memory_size =
4066         nat_calc_bihash_memory (sm->translation_buckets,
4067                                 sizeof (clib_bihash_16_8_t));
4068     }
4069   sm->translation_memory_size = translation_memory_size;
4070   if (0 == user_memory_size)
4071     {
4072       user_memory_size =
4073         nat_calc_bihash_memory (sm->max_users_per_thread,
4074                                 sizeof (clib_bihash_8_8_t));
4075     }
4076   sm->user_memory_size = user_memory_size;
4077   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
4078
4079   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4080     sm->max_translations_per_thread : max_translations_per_user;
4081
4082   sm->outside_vrf_id = outside_vrf_id;
4083   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4084                                                              outside_vrf_id,
4085                                                              nat_fib_src_hi);
4086   nm->outside_vrf_id = outside_ip6_vrf_id;
4087   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
4088                                                              outside_ip6_vrf_id,
4089                                                              nat_fib_src_hi);
4090   sm->inside_vrf_id = inside_vrf_id;
4091   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4092                                                             inside_vrf_id,
4093                                                             nat_fib_src_hi);
4094   sm->static_mapping_only = static_mapping_only;
4095   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4096
4097   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4098                   nat64_st_memory_size);
4099
4100   if (sm->deterministic)
4101     {
4102       sm->in2out_node_index = snat_det_in2out_node.index;
4103       sm->in2out_output_node_index = ~0;
4104       sm->out2in_node_index = snat_det_out2in_node.index;
4105       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4106       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4107     }
4108   else
4109     {
4110       if (sm->endpoint_dependent)
4111         {
4112           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4113           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4114
4115           sm->in2out_node_index = nat44_ed_in2out_node.index;
4116           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4117           sm->out2in_node_index = nat44_ed_out2in_node.index;
4118
4119           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4120           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4121           nat_affinity_init (vm);
4122           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4123                        nat_ha_sref_ed_cb);
4124           clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4125                                  sm->translation_buckets,
4126                                  translation_memory_size);
4127           clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4128                                               format_ed_session_kvp);
4129         }
4130       else
4131         {
4132           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4133           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4134
4135           sm->in2out_node_index = snat_in2out_node.index;
4136           sm->in2out_output_node_index = snat_in2out_output_node.index;
4137           sm->out2in_node_index = snat_out2in_node.index;
4138
4139           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4140           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4141           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4142         }
4143       if (!static_mapping_only ||
4144           (static_mapping_only && static_mapping_connection_tracking))
4145         {
4146           /* *INDENT-OFF* */
4147           vec_foreach (tsm, sm->per_thread_data)
4148             {
4149               nat44_db_init (tsm);
4150             }
4151           /* *INDENT-ON* */
4152         }
4153       else
4154         {
4155           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4156           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4157         }
4158       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4159                             "static_mapping_by_local", static_mapping_buckets,
4160                             static_mapping_memory_size);
4161       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4162                                          format_static_mapping_kvp);
4163
4164       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4165                             "static_mapping_by_external",
4166                             static_mapping_buckets,
4167                             static_mapping_memory_size);
4168       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4169                                          format_static_mapping_kvp);
4170     }
4171
4172   return 0;
4173 }
4174
4175 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4176
4177 static void
4178 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4179                                  uword opaque,
4180                                  u32 sw_if_index,
4181                                  ip4_address_t * address,
4182                                  u32 address_length,
4183                                  u32 if_address_index, u32 is_delete)
4184 {
4185   snat_main_t *sm = &snat_main;
4186   snat_static_map_resolve_t *rp;
4187   snat_static_mapping_t *m;
4188   clib_bihash_kv_8_8_t kv, value;
4189   int i, rv;
4190   ip4_address_t l_addr;
4191
4192   for (i = 0; i < vec_len (sm->to_resolve); i++)
4193     {
4194       rp = sm->to_resolve + i;
4195       if (rp->addr_only == 0)
4196         continue;
4197       if (rp->sw_if_index == sw_if_index)
4198         goto match;
4199     }
4200
4201   return;
4202
4203 match:
4204   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
4205               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
4206   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4207     m = 0;
4208   else
4209     m = pool_elt_at_index (sm->static_mappings, value.value);
4210
4211   if (!is_delete)
4212     {
4213       /* Don't trip over lease renewal, static config */
4214       if (m)
4215         return;
4216     }
4217   else
4218     {
4219       if (!m)
4220         return;
4221     }
4222
4223   /* Indetity mapping? */
4224   if (rp->l_addr.as_u32 == 0)
4225     l_addr.as_u32 = address[0].as_u32;
4226   else
4227     l_addr.as_u32 = rp->l_addr.as_u32;
4228   /* Add the static mapping */
4229   rv = snat_add_static_mapping (l_addr,
4230                                 address[0],
4231                                 rp->l_port,
4232                                 rp->e_port,
4233                                 rp->vrf_id,
4234                                 rp->addr_only, ~0 /* sw_if_index */ ,
4235                                 rp->proto, !is_delete, rp->twice_nat,
4236                                 rp->out2in_only, rp->tag, rp->identity_nat);
4237   if (rv)
4238     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4239 }
4240
4241 static void
4242 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4243                                        uword opaque,
4244                                        u32 sw_if_index,
4245                                        ip4_address_t * address,
4246                                        u32 address_length,
4247                                        u32 if_address_index, u32 is_delete)
4248 {
4249   snat_main_t *sm = &snat_main;
4250   snat_static_map_resolve_t *rp;
4251   ip4_address_t l_addr;
4252   int i, j;
4253   int rv;
4254   u8 twice_nat = 0;
4255   snat_address_t *addresses = sm->addresses;
4256
4257   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4258     {
4259       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4260         goto match;
4261     }
4262
4263   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4264     {
4265       twice_nat = 1;
4266       addresses = sm->twice_nat_addresses;
4267       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4268         goto match;
4269     }
4270
4271   return;
4272
4273 match:
4274   if (!is_delete)
4275     {
4276       /* Don't trip over lease renewal, static config */
4277       for (j = 0; j < vec_len (addresses); j++)
4278         if (addresses[j].addr.as_u32 == address->as_u32)
4279           return;
4280
4281       (void) snat_add_address (sm, address, ~0, twice_nat);
4282       /* Scan static map resolution vector */
4283       for (j = 0; j < vec_len (sm->to_resolve); j++)
4284         {
4285           rp = sm->to_resolve + j;
4286           if (rp->addr_only)
4287             continue;
4288           /* On this interface? */
4289           if (rp->sw_if_index == sw_if_index)
4290             {
4291               /* Indetity mapping? */
4292               if (rp->l_addr.as_u32 == 0)
4293                 l_addr.as_u32 = address[0].as_u32;
4294               else
4295                 l_addr.as_u32 = rp->l_addr.as_u32;
4296               /* Add the static mapping */
4297               rv = snat_add_static_mapping (l_addr,
4298                                             address[0],
4299                                             rp->l_port,
4300                                             rp->e_port,
4301                                             rp->vrf_id,
4302                                             rp->addr_only,
4303                                             ~0 /* sw_if_index */ ,
4304                                             rp->proto,
4305                                             rp->is_add, rp->twice_nat,
4306                                             rp->out2in_only, rp->tag,
4307                                             rp->identity_nat);
4308               if (rv)
4309                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4310                                     "i4", rv);
4311             }
4312         }
4313       return;
4314     }
4315   else
4316     {
4317       (void) snat_del_address (sm, address[0], 1, twice_nat);
4318       return;
4319     }
4320 }
4321
4322
4323 int
4324 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4325                             u8 twice_nat)
4326 {
4327   ip4_main_t *ip4_main = sm->ip4_main;
4328   ip4_address_t *first_int_addr;
4329   snat_static_map_resolve_t *rp;
4330   u32 *indices_to_delete = 0;
4331   int i, j;
4332   u32 *auto_add_sw_if_indices =
4333     twice_nat ? sm->
4334     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4335
4336   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4337     );
4338
4339   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4340     {
4341       if (auto_add_sw_if_indices[i] == sw_if_index)
4342         {
4343           if (is_del)
4344             {
4345               /* if have address remove it */
4346               if (first_int_addr)
4347                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4348               else
4349                 {
4350                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4351                     {
4352                       rp = sm->to_resolve + j;
4353                       if (rp->sw_if_index == sw_if_index)
4354                         vec_add1 (indices_to_delete, j);
4355                     }
4356                   if (vec_len (indices_to_delete))
4357                     {
4358                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4359                         vec_del1 (sm->to_resolve, j);
4360                       vec_free (indices_to_delete);
4361                     }
4362                 }
4363               if (twice_nat)
4364                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4365               else
4366                 vec_del1 (sm->auto_add_sw_if_indices, i);
4367             }
4368           else
4369             return VNET_API_ERROR_VALUE_EXIST;
4370
4371           return 0;
4372         }
4373     }
4374
4375   if (is_del)
4376     return VNET_API_ERROR_NO_SUCH_ENTRY;
4377
4378   /* add to the auto-address list */
4379   if (twice_nat)
4380     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4381   else
4382     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4383
4384   /* If the address is already bound - or static - add it now */
4385   if (first_int_addr)
4386     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4387
4388   return 0;
4389 }
4390
4391 int
4392 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4393                    nat_protocol_t proto, u32 vrf_id, int is_in)
4394 {
4395   snat_main_per_thread_data_t *tsm;
4396   clib_bihash_kv_8_8_t kv, value;
4397   ip4_header_t ip;
4398   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4399   snat_session_t *s;
4400   clib_bihash_8_8_t *t;
4401
4402   if (sm->endpoint_dependent)
4403     return VNET_API_ERROR_UNSUPPORTED;
4404
4405   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4406   if (sm->num_workers > 1)
4407     tsm =
4408       vec_elt_at_index (sm->per_thread_data,
4409                         sm->worker_in2out_cb (&ip, fib_index, 0));
4410   else
4411     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4412
4413   init_nat_k (&kv, *addr, port, fib_index, proto);
4414   t = is_in ? &tsm->in2out : &tsm->out2in;
4415   if (!clib_bihash_search_8_8 (t, &kv, &value))
4416     {
4417       if (pool_is_free_index (tsm->sessions, value.value))
4418         return VNET_API_ERROR_UNSPECIFIED;
4419
4420       s = pool_elt_at_index (tsm->sessions, value.value);
4421       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4422       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4423       return 0;
4424     }
4425
4426   return VNET_API_ERROR_NO_SUCH_ENTRY;
4427 }
4428
4429 int
4430 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4431                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4432                       u32 vrf_id, int is_in)
4433 {
4434   ip4_header_t ip;
4435   clib_bihash_16_8_t *t;
4436   clib_bihash_kv_16_8_t kv, value;
4437   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4438   snat_session_t *s;
4439   snat_main_per_thread_data_t *tsm;
4440
4441   if (!sm->endpoint_dependent)
4442     return VNET_API_ERROR_FEATURE_DISABLED;
4443
4444   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4445   if (sm->num_workers > 1)
4446     tsm =
4447       vec_elt_at_index (sm->per_thread_data,
4448                         sm->worker_in2out_cb (&ip, fib_index, 0));
4449   else
4450     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4451
4452   t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
4453   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
4454   if (clib_bihash_search_16_8 (t, &kv, &value))
4455     {
4456       return VNET_API_ERROR_NO_SUCH_ENTRY;
4457     }
4458
4459   if (pool_is_free_index (tsm->sessions, value.value))
4460     return VNET_API_ERROR_UNSPECIFIED;
4461   s = pool_elt_at_index (tsm->sessions, value.value);
4462   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4463   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
4464   return 0;
4465 }
4466
4467 void
4468 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4469 {
4470   snat_main_t *sm = &snat_main;
4471
4472   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4473   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4474   sm->psid = psid;
4475   sm->psid_offset = psid_offset;
4476   sm->psid_length = psid_length;
4477 }
4478
4479 void
4480 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4481 {
4482   snat_main_t *sm = &snat_main;
4483
4484   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4485   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4486   sm->start_port = start_port;
4487   sm->end_port = end_port;
4488 }
4489
4490 void
4491 nat_set_alloc_addr_and_port_default (void)
4492 {
4493   snat_main_t *sm = &snat_main;
4494
4495   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4496   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4497 }
4498
4499 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4500                                  vlib_node_runtime_t * node,
4501                                  vlib_frame_t * frame)
4502 {
4503   return 0;
4504 }
4505
4506 /* *INDENT-OFF* */
4507 VLIB_REGISTER_NODE (nat_default_node) = {
4508   .name = "nat-default",
4509   .vector_size = sizeof (u32),
4510   .format_trace = 0,
4511   .type = VLIB_NODE_TYPE_INTERNAL,
4512   .n_errors = 0,
4513   .n_next_nodes = NAT_N_NEXT,
4514   .next_nodes = {
4515     [NAT_NEXT_DROP] = "error-drop",
4516     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4517     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4518     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4519     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4520     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4521     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4522     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4523     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4524   },
4525 };
4526 /* *INDENT-ON* */
4527
4528 /*
4529  * fd.io coding-style-patch-verification: ON
4530  *
4531  * Local Variables:
4532  * eval: (c-set-style "gnu")
4533  * End:
4534  */