src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat44/inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36 #include <vppinfra/bihash_16_8.h>
  37 #include <nat/nat44/ed_inlines.h>
  38
  39 #include <vpp/app/version.h>
  40
  41 snat_main_t snat_main;
  42
  43 fib_source_t nat_fib_src_hi;
  44 fib_source_t nat_fib_src_low;
  45
  46 /* *INDENT-OFF* */
  47 /* Hook up input features */
  48 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat-pre-in2out",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-sv-reassembly-feature"),
  53 };
  54 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat-pre-out2in",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  58                                "ip4-dhcp-client-detect",
  59                                "ip4-sv-reassembly-feature"),
  60 };
  61 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  62   .arc_name = "ip4-unicast",
  63   .node_name = "nat44-in2out-worker-handoff",
  64   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  65 };
  66 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  67   .arc_name = "ip4-unicast",
  68   .node_name = "nat44-out2in-worker-handoff",
  69   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  70                                "ip4-dhcp-client-detect"),
  71 };
  72 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  73   .arc_name = "ip4-unicast",
  74   .node_name = "nat44-in2out",
  75   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  76 };
  77 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  78   .arc_name = "ip4-unicast",
  79   .node_name = "nat44-out2in",
  80   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  81                                "ip4-dhcp-client-detect"),
  82 };
  83 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-det-in2out",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  92 };
  93 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  94   .arc_name = "ip4-unicast",
  95   .node_name = "nat44-det-out2in",
  96   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  97                                "ip4-dhcp-client-detect"),
  98 };
  99 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
 100   .arc_name = "ip4-unicast",
 101   .node_name = "nat44-det-classify",
 102   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-ed-in2out",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 108 };
 109 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-ed-out2in",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-classify",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-handoff-classify",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 124 };
 125 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 126   .arc_name = "ip4-unicast",
 127   .node_name = "nat44-in2out-fast",
 128   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 129 };
 130 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 131   .arc_name = "ip4-unicast",
 132   .node_name = "nat44-out2in-fast",
 133   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 134                                "ip4-dhcp-client-detect"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 137   .arc_name = "ip4-unicast",
 138   .node_name = "nat44-hairpin-dst",
 139   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 140 };
 141 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 142   .arc_name = "ip4-unicast",
 143   .node_name = "nat44-ed-hairpin-dst",
 144   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 145 };
 146
 147 /* Hook up output features */
 148 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-in2out-output",
 151   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 152 };
 153 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 154   .arc_name = "ip4-output",
 155   .node_name = "nat44-in2out-output-worker-handoff",
 156   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 157 };
 158 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 159   .arc_name = "ip4-output",
 160   .node_name = "nat44-hairpin-src",
 161   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 164   .arc_name = "ip4-output",
 165   .node_name = "nat44-ed-in2out-output",
 166   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 167   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 168 };
 169 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 170   .arc_name = "ip4-output",
 171   .node_name = "nat44-ed-hairpin-src",
 172   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 173   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 174 };
 175
 176 /* Hook up ip4-local features */
 177 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 178 {
 179   .arc_name = "ip4-local",
 180   .node_name = "nat44-hairpinning",
 181   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 182 };
 183 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 184 {
 185   .arc_name = "ip4-local",
 186   .node_name = "nat44-ed-hairpinning",
 187   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 188 };
 189
 190
 191 VLIB_PLUGIN_REGISTER () = {
 192     .version = VPP_BUILD_VER,
 193     .description = "Network Address Translation (NAT)",
 194 };
 195 /* *INDENT-ON* */
 196
 197 void
 198 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 199                        u8 is_ha)
 200 {
 201   clib_bihash_kv_8_8_t kv;
 202   u8 proto;
 203   u16 r_port, l_port;
 204   ip4_address_t *l_addr, *r_addr;
 205   u32 fib_index = 0;
 206   clib_bihash_kv_16_8_t ed_kv;
 207   snat_main_per_thread_data_t *tsm =
 208     vec_elt_at_index (sm->per_thread_data, thread_index);
 209
 210   if (is_fwd_bypass_session (s))
 211     {
 212       if (snat_is_unk_proto_session (s))
 213         {
 214           init_ed_k (&ed_kv, s->in2out.addr, 0, s->ext_host_addr, 0, 0,
 215                      s->in2out.port);
 216         }
 217       else
 218         {
 219           l_port = s->in2out.port;
 220           r_port = s->ext_host_port;
 221           l_addr = &s->in2out.addr;
 222           r_addr = &s->ext_host_addr;
 223           proto = nat_proto_to_ip_proto (s->nat_proto);
 224           init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index,
 225                      proto);
 226         }
 227       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 228         nat_elog_warn ("in2out_ed key del failed");
 229       return;
 230     }
 231
 232   /* session lookup tables */
 233   if (is_ed_session (s))
 234     {
 235       if (is_affinity_sessions (s))
 236         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 237                              s->nat_proto, s->out2in.port);
 238       l_addr = &s->out2in.addr;
 239       r_addr = &s->ext_host_addr;
 240       fib_index = s->out2in.fib_index;
 241       if (snat_is_unk_proto_session (s))
 242         {
 243           proto = s->in2out.port;
 244           r_port = 0;
 245           l_port = 0;
 246         }
 247       else
 248         {
 249           proto = nat_proto_to_ip_proto (s->nat_proto);
 250           l_port = s->out2in.port;
 251           r_port = s->ext_host_port;
 252         }
 253       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 254       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 255         nat_elog_warn ("out2in_ed key del failed");
 256       l_addr = &s->in2out.addr;
 257       fib_index = s->in2out.fib_index;
 258       if (!snat_is_unk_proto_session (s))
 259         l_port = s->in2out.port;
 260       if (is_twice_nat_session (s))
 261         {
 262           r_addr = &s->ext_host_nat_addr;
 263           r_port = s->ext_host_nat_port;
 264         }
 265       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 266       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 267         nat_elog_warn ("in2out_ed key del failed");
 268
 269       if (!is_ha)
 270         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 271                                &s->in2out.addr, s->in2out.port,
 272                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 273                                &s->out2in.addr, s->out2in.port,
 274                                &s->ext_host_addr, s->ext_host_port,
 275                                s->nat_proto, is_twice_nat_session (s));
 276     }
 277   else
 278     {
 279       init_nat_i2o_k (&kv, s);
 280       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 281         nat_elog_warn ("in2out key del failed");
 282       init_nat_o2i_k (&kv, s);
 283       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 284         nat_elog_warn ("out2in key del failed");
 285
 286       if (!is_ha)
 287         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 288                                  &s->in2out.addr, s->in2out.port,
 289                                  &s->out2in.addr, s->out2in.port,
 290                                  s->nat_proto);
 291     }
 292
 293   if (snat_is_unk_proto_session (s))
 294     return;
 295
 296   if (!is_ha)
 297     {
 298       /* log NAT event */
 299       snat_ipfix_logging_nat44_ses_delete (thread_index,
 300                                            s->in2out.addr.as_u32,
 301                                            s->out2in.addr.as_u32,
 302                                            s->nat_proto,
 303                                            s->in2out.port,
 304                                            s->out2in.port,
 305                                            s->in2out.fib_index);
 306
 307       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 308                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 309                    thread_index);
 310     }
 311
 312   /* Twice NAT address and port for external host */
 313   if (is_twice_nat_session (s))
 314     {
 315       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 316                                           thread_index,
 317                                           &s->ext_host_nat_addr,
 318                                           s->ext_host_nat_port, s->nat_proto);
 319     }
 320
 321   if (snat_is_session_static (s))
 322     return;
 323
 324   snat_free_outside_address_and_port (sm->addresses, thread_index,
 325                                       &s->out2in.addr, s->out2in.port,
 326                                       s->nat_proto);
 327 }
 328
 329 int
 330 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
 331 {
 332   snat_main_t *sm = &snat_main;
 333   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
 334   u32 len = vec_len (sm->max_translations_per_fib);
 335
 336   if (len <= fib_index)
 337     {
 338       vec_validate (sm->max_translations_per_fib, fib_index + 1);
 339
 340       for (; len < vec_len (sm->max_translations_per_fib); len++)
 341         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
 342     }
 343
 344   sm->max_translations_per_fib[fib_index] = session_limit;
 345   return 0;
 346 }
 347
 348 void
 349 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 350                          u32 thread_index, u8 is_ha)
 351 {
 352   u8 proto;
 353   u16 r_port, l_port;
 354   ip4_address_t *l_addr, *r_addr;
 355   u32 fib_index;
 356   clib_bihash_kv_16_8_t ed_kv;
 357   snat_main_per_thread_data_t *tsm =
 358     vec_elt_at_index (sm->per_thread_data, thread_index);
 359
 360   if (is_fwd_bypass_session (s))
 361     {
 362       if (snat_is_unk_proto_session (s))
 363         {
 364           proto = s->in2out.port;
 365           r_port = 0;
 366           l_port = 0;
 367         }
 368       else
 369         {
 370           proto = nat_proto_to_ip_proto (s->nat_proto);
 371           l_port = s->in2out.port;
 372           r_port = s->ext_host_port;
 373         }
 374
 375       l_addr = &s->in2out.addr;
 376       r_addr = &s->ext_host_addr;
 377       fib_index = 0;
 378       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 379
 380       if (PREDICT_FALSE
 381           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 382         nat_elog_warn ("in2out_ed key del failed");
 383       return;
 384     }
 385
 386   /* session lookup tables */
 387   if (is_affinity_sessions (s))
 388     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 389                          s->nat_proto, s->out2in.port);
 390   l_addr = &s->out2in.addr;
 391   r_addr = &s->ext_host_addr;
 392   fib_index = s->out2in.fib_index;
 393   if (snat_is_unk_proto_session (s))
 394     {
 395       proto = s->in2out.port;
 396       r_port = 0;
 397       l_port = 0;
 398     }
 399   else
 400     {
 401       proto = nat_proto_to_ip_proto (s->nat_proto);
 402       l_port = s->out2in.port;
 403       r_port = s->ext_host_port;
 404     }
 405   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 406
 407   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0)))
 408     nat_elog_warn ("out2in_ed key del failed");
 409
 410   l_addr = &s->in2out.addr;
 411   fib_index = s->in2out.fib_index;
 412
 413   if (!snat_is_unk_proto_session (s))
 414     l_port = s->in2out.port;
 415
 416   if (is_twice_nat_session (s))
 417     {
 418       r_addr = &s->ext_host_nat_addr;
 419       r_port = s->ext_host_nat_port;
 420     }
 421   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 422
 423   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 424     nat_elog_warn ("in2out_ed key del failed");
 425
 426   if (!is_ha)
 427     {
 428       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 429                              &s->in2out.addr, s->in2out.port,
 430                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 431                              &s->out2in.addr, s->out2in.port,
 432                              &s->ext_host_addr, s->ext_host_port,
 433                              s->nat_proto, is_twice_nat_session (s));
 434     }
 435
 436   if (snat_is_unk_proto_session (s))
 437     return;
 438
 439   if (!is_ha)
 440     {
 441       snat_ipfix_logging_nat44_ses_delete (thread_index,
 442                                            s->in2out.addr.as_u32,
 443                                            s->out2in.addr.as_u32,
 444                                            s->nat_proto,
 445                                            s->in2out.port,
 446                                            s->out2in.port,
 447                                            s->in2out.fib_index);
 448       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 449                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 450                    thread_index);
 451     }
 452
 453   /* Twice NAT address and port for external host */
 454   if (is_twice_nat_session (s))
 455     {
 456       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 457                                           thread_index,
 458                                           &s->ext_host_nat_addr,
 459                                           s->ext_host_nat_port, s->nat_proto);
 460     }
 461
 462   if (snat_is_session_static (s))
 463     return;
 464
 465   snat_free_outside_address_and_port (sm->addresses, thread_index,
 466                                       &s->out2in.addr, s->out2in.port,
 467                                       s->nat_proto);
 468 }
 469
 470
 471 snat_user_t *
 472 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 473                         u32 thread_index)
 474 {
 475   snat_user_t *u = 0;
 476   snat_user_key_t user_key;
 477   clib_bihash_kv_8_8_t kv, value;
 478   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 479   dlist_elt_t *per_user_list_head_elt;
 480
 481   user_key.addr.as_u32 = addr->as_u32;
 482   user_key.fib_index = fib_index;
 483   kv.key = user_key.as_u64;
 484
 485   /* Ever heard of the "user" = src ip4 address before? */
 486   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 487     {
 488       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 489         {
 490           vlib_increment_simple_counter (&sm->user_limit_reached,
 491                                          thread_index, 0, 1);
 492           nat_elog_warn ("maximum user limit reached");
 493           return NULL;
 494         }
 495       /* no, make a new one */
 496       pool_get (tsm->users, u);
 497       clib_memset (u, 0, sizeof (*u));
 498
 499       u->addr.as_u32 = addr->as_u32;
 500       u->fib_index = fib_index;
 501
 502       pool_get (tsm->list_pool, per_user_list_head_elt);
 503
 504       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 505         tsm->list_pool;
 506
 507       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 508
 509       kv.value = u - tsm->users;
 510
 511       /* add user */
 512       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 513         {
 514           nat_elog_warn ("user_hash key add failed");
 515           nat44_delete_user_with_no_session (sm, u, thread_index);
 516           return NULL;
 517         }
 518
 519       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 520                                pool_elts (tsm->users));
 521     }
 522   else
 523     {
 524       u = pool_elt_at_index (tsm->users, value.value);
 525     }
 526
 527   return u;
 528 }
 529
 530 snat_session_t *
 531 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 532                               u32 thread_index, f64 now)
 533 {
 534   snat_session_t *s;
 535   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 536   u32 oldest_per_user_translation_list_index, session_index;
 537   dlist_elt_t *oldest_per_user_translation_list_elt;
 538   dlist_elt_t *per_user_translation_list_elt;
 539
 540   /* Over quota? Recycle the least recently used translation */
 541   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 542     {
 543       oldest_per_user_translation_list_index =
 544         clib_dlist_remove_head (tsm->list_pool,
 545                                 u->sessions_per_user_list_head_index);
 546
 547       ASSERT (oldest_per_user_translation_list_index != ~0);
 548
 549       /* Add it back to the end of the LRU list */
 550       clib_dlist_addtail (tsm->list_pool,
 551                           u->sessions_per_user_list_head_index,
 552                           oldest_per_user_translation_list_index);
 553       /* Get the list element */
 554       oldest_per_user_translation_list_elt =
 555         pool_elt_at_index (tsm->list_pool,
 556                            oldest_per_user_translation_list_index);
 557
 558       /* Get the session index from the list element */
 559       session_index = oldest_per_user_translation_list_elt->value;
 560
 561       /* Get the session */
 562       s = pool_elt_at_index (tsm->sessions, session_index);
 563       nat_free_session_data (sm, s, thread_index, 0);
 564       if (snat_is_session_static (s))
 565         u->nstaticsessions--;
 566       else
 567         u->nsessions--;
 568       s->flags = 0;
 569       s->total_bytes = 0;
 570       s->total_pkts = 0;
 571       s->state = 0;
 572       s->ext_host_addr.as_u32 = 0;
 573       s->ext_host_port = 0;
 574       s->ext_host_nat_addr.as_u32 = 0;
 575       s->ext_host_nat_port = 0;
 576     }
 577   else
 578     {
 579       pool_get (tsm->sessions, s);
 580       clib_memset (s, 0, sizeof (*s));
 581
 582       /* Create list elts */
 583       pool_get (tsm->list_pool, per_user_translation_list_elt);
 584       clib_dlist_init (tsm->list_pool,
 585                        per_user_translation_list_elt - tsm->list_pool);
 586
 587       per_user_translation_list_elt->value = s - tsm->sessions;
 588       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 589       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 590
 591       clib_dlist_addtail (tsm->list_pool,
 592                           s->per_user_list_head_index,
 593                           per_user_translation_list_elt - tsm->list_pool);
 594
 595       s->user_index = u - tsm->users;
 596       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 597                                pool_elts (tsm->sessions));
 598     }
 599
 600   s->ha_last_refreshed = now;
 601
 602   return s;
 603 }
 604
 605 void
 606 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 607                           int is_add)
 608 {
 609   fib_prefix_t prefix = {
 610     .fp_len = p_len,
 611     .fp_proto = FIB_PROTOCOL_IP4,
 612     .fp_addr = {
 613                 .ip4.as_u32 = addr->as_u32,
 614                 },
 615   };
 616   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 617
 618   if (is_add)
 619     fib_table_entry_update_one_path (fib_index,
 620                                      &prefix,
 621                                      nat_fib_src_low,
 622                                      (FIB_ENTRY_FLAG_CONNECTED |
 623                                       FIB_ENTRY_FLAG_LOCAL |
 624                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 625                                      DPO_PROTO_IP4,
 626                                      NULL,
 627                                      sw_if_index,
 628                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 629   else
 630     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 631 }
 632
 633 int
 634 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 635                   u8 twice_nat)
 636 {
 637   snat_address_t *ap;
 638   snat_interface_t *i;
 639   vlib_thread_main_t *tm = vlib_get_thread_main ();
 640
 641   if (twice_nat && !sm->endpoint_dependent)
 642     return VNET_API_ERROR_FEATURE_DISABLED;
 643
 644   /* Check if address already exists */
 645   /* *INDENT-OFF* */
 646   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 647     {
 648       if (ap->addr.as_u32 == addr->as_u32)
 649         return VNET_API_ERROR_VALUE_EXIST;
 650     }
 651   /* *INDENT-ON* */
 652
 653   if (twice_nat)
 654     vec_add2 (sm->twice_nat_addresses, ap, 1);
 655   else
 656     vec_add2 (sm->addresses, ap, 1);
 657
 658   ap->addr = *addr;
 659   if (vrf_id != ~0)
 660     ap->fib_index =
 661       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 662                                          nat_fib_src_low);
 663   else
 664     ap->fib_index = ~0;
 665 #define _(N, i, n, s) \
 666   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 667   ap->busy_##n##_ports = 0; \
 668   ap->busy_##n##_ports_per_thread = 0;\
 669   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 670   foreach_nat_protocol
 671 #undef _
 672     if (twice_nat)
 673     return 0;
 674
 675   /* Add external address to FIB */
 676   /* *INDENT-OFF* */
 677   pool_foreach (i, sm->interfaces,
 678   ({
 679     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 680       continue;
 681
 682     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 683     break;
 684   }));
 685   pool_foreach (i, sm->output_feature_interfaces,
 686   ({
 687     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 688       continue;
 689
 690     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 691     break;
 692   }));
 693   /* *INDENT-ON* */
 694
 695   return 0;
 696 }
 697
 698 static int
 699 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 700 {
 701   snat_static_mapping_t *m;
 702   /* *INDENT-OFF* */
 703   pool_foreach (m, sm->static_mappings,
 704   ({
 705       if (is_addr_only_static_mapping (m) ||
 706           is_out2in_only_static_mapping (m) ||
 707           is_identity_static_mapping (m))
 708         continue;
 709       if (m->external_addr.as_u32 == addr.as_u32)
 710         return 1;
 711   }));
 712   /* *INDENT-ON* */
 713
 714   return 0;
 715 }
 716
 717 static void
 718 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 719                                        ip4_address_t l_addr,
 720                                        u16 l_port,
 721                                        u32 sw_if_index,
 722                                        u16 e_port,
 723                                        u32 vrf_id,
 724                                        nat_protocol_t proto,
 725                                        int addr_only, int is_add, u8 * tag,
 726                                        int twice_nat, int out2in_only,
 727                                        int identity_nat)
 728 {
 729   snat_static_map_resolve_t *rp;
 730
 731   vec_add2 (sm->to_resolve, rp, 1);
 732   rp->l_addr.as_u32 = l_addr.as_u32;
 733   rp->l_port = l_port;
 734   rp->sw_if_index = sw_if_index;
 735   rp->e_port = e_port;
 736   rp->vrf_id = vrf_id;
 737   rp->proto = proto;
 738   rp->addr_only = addr_only;
 739   rp->is_add = is_add;
 740   rp->twice_nat = twice_nat;
 741   rp->out2in_only = out2in_only;
 742   rp->identity_nat = identity_nat;
 743   rp->tag = vec_dup (tag);
 744 }
 745
 746 static u32
 747 get_thread_idx_by_port (u16 e_port)
 748 {
 749   snat_main_t *sm = &snat_main;
 750   u32 thread_idx = sm->num_workers;
 751   if (sm->num_workers > 1)
 752     {
 753       thread_idx =
 754         sm->first_worker_index +
 755         sm->workers[(e_port - 1024) / sm->port_per_thread];
 756     }
 757   return thread_idx;
 758 }
 759
 760 void
 761 snat_static_mapping_del_sessions (snat_main_t * sm,
 762                                   snat_main_per_thread_data_t * tsm,
 763                                   snat_user_key_t u_key, int addr_only,
 764                                   ip4_address_t e_addr, u16 e_port)
 765 {
 766   clib_bihash_kv_8_8_t kv, value;
 767   kv.key = u_key.as_u64;
 768   u64 user_index;
 769   dlist_elt_t *head, *elt;
 770   snat_user_t *u;
 771   snat_session_t *s;
 772   u32 elt_index, head_index, ses_index;
 773   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 774     {
 775       user_index = value.value;
 776       u = pool_elt_at_index (tsm->users, user_index);
 777       if (u->nstaticsessions)
 778         {
 779           head_index = u->sessions_per_user_list_head_index;
 780           head = pool_elt_at_index (tsm->list_pool, head_index);
 781           elt_index = head->next;
 782           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 783           ses_index = elt->value;
 784           while (ses_index != ~0)
 785             {
 786               s = pool_elt_at_index (tsm->sessions, ses_index);
 787               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 788               ses_index = elt->value;
 789
 790               if (!addr_only)
 791                 {
 792                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 793                       (s->out2in.port != e_port))
 794                     continue;
 795                 }
 796
 797               if (is_lb_session (s))
 798                 continue;
 799
 800               if (!snat_is_session_static (s))
 801                 continue;
 802
 803               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 804               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 805
 806               if (!addr_only)
 807                 break;
 808             }
 809         }
 810     }
 811 }
 812
 813 void
 814 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 815                                      snat_main_per_thread_data_t * tsm,
 816                                      ip4_address_t l_addr,
 817                                      u16 l_port,
 818                                      u8 protocol,
 819                                      u32 fib_index, int addr_only,
 820                                      ip4_address_t e_addr, u16 e_port)
 821 {
 822   snat_session_t *s;
 823   u32 *indexes_to_free = NULL;
 824   /* *INDENT-OFF* */
 825   pool_foreach (s, tsm->sessions, {
 826     if (s->in2out.fib_index != fib_index ||
 827         s->in2out.addr.as_u32 != l_addr.as_u32)
 828       {
 829         continue;
 830       }
 831     if (!addr_only)
 832       {
 833         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 834             s->out2in.port != e_port ||
 835             s->in2out.port != l_port ||
 836             s->nat_proto != protocol)
 837           continue;
 838       }
 839
 840     if (is_lb_session (s))
 841       continue;
 842     if (!snat_is_session_static (s))
 843       continue;
 844     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 845     vec_add1 (indexes_to_free, s - tsm->sessions);
 846     if (!addr_only)
 847       break;
 848   });
 849   /* *INDENT-ON* */
 850   u32 *ses_index;
 851   vec_foreach (ses_index, indexes_to_free)
 852   {
 853     s = pool_elt_at_index (tsm->sessions, *ses_index);
 854     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 855   }
 856   vec_free (indexes_to_free);
 857 }
 858
 859 int
 860 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 861                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 862                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 863                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 864                          u8 identity_nat)
 865 {
 866   snat_main_t *sm = &snat_main;
 867   snat_static_mapping_t *m;
 868   clib_bihash_kv_8_8_t kv, value;
 869   snat_address_t *a = 0;
 870   u32 fib_index = ~0;
 871   snat_interface_t *interface;
 872   int i;
 873   snat_main_per_thread_data_t *tsm;
 874   snat_user_key_t u_key;
 875   snat_user_t *u;
 876   dlist_elt_t *head, *elt;
 877   u32 elt_index, head_index;
 878   u32 ses_index;
 879   u64 user_index;
 880   snat_session_t *s;
 881   snat_static_map_resolve_t *rp, *rp_match = 0;
 882   nat44_lb_addr_port_t *local;
 883   u32 find = ~0;
 884
 885   if (!sm->endpoint_dependent)
 886     {
 887       if (twice_nat || out2in_only)
 888         return VNET_API_ERROR_FEATURE_DISABLED;
 889     }
 890
 891   /* If the external address is a specific interface address */
 892   if (sw_if_index != ~0)
 893     {
 894       ip4_address_t *first_int_addr;
 895
 896       for (i = 0; i < vec_len (sm->to_resolve); i++)
 897         {
 898           rp = sm->to_resolve + i;
 899           if (rp->sw_if_index != sw_if_index ||
 900               rp->l_addr.as_u32 != l_addr.as_u32 ||
 901               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 902             continue;
 903
 904           if (!addr_only)
 905             {
 906               if ((rp->l_port != l_port && rp->e_port != e_port)
 907                   || rp->proto != proto)
 908                 continue;
 909             }
 910
 911           rp_match = rp;
 912           break;
 913         }
 914
 915       /* Might be already set... */
 916       first_int_addr = ip4_interface_first_address
 917         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 918
 919       if (is_add)
 920         {
 921           if (rp_match)
 922             return VNET_API_ERROR_VALUE_EXIST;
 923
 924           snat_add_static_mapping_when_resolved
 925             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 926              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 927
 928           /* DHCP resolution required? */
 929           if (first_int_addr == 0)
 930             {
 931               return 0;
 932             }
 933           else
 934             {
 935               e_addr.as_u32 = first_int_addr->as_u32;
 936               /* Identity mapping? */
 937               if (l_addr.as_u32 == 0)
 938                 l_addr.as_u32 = e_addr.as_u32;
 939             }
 940         }
 941       else
 942         {
 943           if (!rp_match)
 944             return VNET_API_ERROR_NO_SUCH_ENTRY;
 945
 946           vec_del1 (sm->to_resolve, i);
 947
 948           if (first_int_addr)
 949             {
 950               e_addr.as_u32 = first_int_addr->as_u32;
 951               /* Identity mapping? */
 952               if (l_addr.as_u32 == 0)
 953                 l_addr.as_u32 = e_addr.as_u32;
 954             }
 955           else
 956             return 0;
 957         }
 958     }
 959
 960   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 961   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 962     m = 0;
 963   else
 964     m = pool_elt_at_index (sm->static_mappings, value.value);
 965
 966   if (is_add)
 967     {
 968       if (m)
 969         {
 970           if (is_identity_static_mapping (m))
 971             {
 972               /* *INDENT-OFF* */
 973               pool_foreach (local, m->locals,
 974               ({
 975                 if (local->vrf_id == vrf_id)
 976                   return VNET_API_ERROR_VALUE_EXIST;
 977               }));
 978               /* *INDENT-ON* */
 979               pool_get (m->locals, local);
 980               local->vrf_id = vrf_id;
 981               local->fib_index =
 982                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 983                                                    nat_fib_src_low);
 984               init_nat_kv (&kv, m->local_addr, m->local_port,
 985                            local->fib_index, m->proto,
 986                            m - sm->static_mappings);
 987               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 988               return 0;
 989             }
 990           else
 991             return VNET_API_ERROR_VALUE_EXIST;
 992         }
 993
 994       if (twice_nat && addr_only)
 995         return VNET_API_ERROR_UNSUPPORTED;
 996
 997       /* Convert VRF id to FIB index */
 998       if (vrf_id != ~0)
 999         fib_index =
1000           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1001                                              nat_fib_src_low);
1002       /* If not specified use inside VRF id from SNAT plugin startup config */
1003       else
1004         {
1005           fib_index = sm->inside_fib_index;
1006           vrf_id = sm->inside_vrf_id;
1007           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1008         }
1009
1010       if (!(out2in_only || identity_nat))
1011         {
1012           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
1013                       addr_only ? 0 : proto);
1014           if (!clib_bihash_search_8_8
1015               (&sm->static_mapping_by_local, &kv, &value))
1016             return VNET_API_ERROR_VALUE_EXIST;
1017         }
1018
1019       /* Find external address in allocated addresses and reserve port for
1020          address and port pair mapping when dynamic translations enabled */
1021       if (!(addr_only || sm->static_mapping_only || out2in_only))
1022         {
1023           for (i = 0; i < vec_len (sm->addresses); i++)
1024             {
1025               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1026                 {
1027                   a = sm->addresses + i;
1028                   /* External port must be unused */
1029                   switch (proto)
1030                     {
1031 #define _(N, j, n, s) \
1032                     case NAT_PROTOCOL_##N: \
1033                       if (a->busy_##n##_port_refcounts[e_port]) \
1034                         return VNET_API_ERROR_INVALID_VALUE; \
1035                       ++a->busy_##n##_port_refcounts[e_port]; \
1036                       if (e_port > 1024) \
1037                         { \
1038                           a->busy_##n##_ports++; \
1039                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1040                         } \
1041                       break;
1042                       foreach_nat_protocol
1043 #undef _
1044                     default:
1045                       nat_elog_info ("unknown protocol");
1046                       return VNET_API_ERROR_INVALID_VALUE_2;
1047                     }
1048                   break;
1049                 }
1050             }
1051           /* External address must be allocated */
1052           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1053             {
1054               if (sw_if_index != ~0)
1055                 {
1056                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1057                     {
1058                       rp = sm->to_resolve + i;
1059                       if (rp->addr_only)
1060                         continue;
1061                       if (rp->sw_if_index != sw_if_index &&
1062                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1063                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1064                           rp->e_port != e_port && rp->proto != proto)
1065                         continue;
1066
1067                       vec_del1 (sm->to_resolve, i);
1068                       break;
1069                     }
1070                 }
1071               return VNET_API_ERROR_NO_SUCH_ENTRY;
1072             }
1073         }
1074
1075       pool_get (sm->static_mappings, m);
1076       clib_memset (m, 0, sizeof (*m));
1077       m->tag = vec_dup (tag);
1078       m->local_addr = l_addr;
1079       m->external_addr = e_addr;
1080       m->twice_nat = twice_nat;
1081       if (out2in_only)
1082         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1083       if (addr_only)
1084         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1085       if (identity_nat)
1086         {
1087           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1088           pool_get (m->locals, local);
1089           local->vrf_id = vrf_id;
1090           local->fib_index = fib_index;
1091         }
1092       else
1093         {
1094           m->vrf_id = vrf_id;
1095           m->fib_index = fib_index;
1096         }
1097       if (!addr_only)
1098         {
1099           m->local_port = l_port;
1100           m->external_port = e_port;
1101           m->proto = proto;
1102         }
1103
1104       if (sm->num_workers > 1)
1105         {
1106           ip4_header_t ip = {
1107             .src_address = m->local_addr,
1108           };
1109           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1110           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1111         }
1112       else
1113         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1114
1115       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
1116                    m - sm->static_mappings);
1117       if (!out2in_only)
1118         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1119
1120       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1121                    m - sm->static_mappings);
1122       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1123
1124       /* Delete dynamic sessions matching local address (+ local port) */
1125       if (!(sm->static_mapping_only))
1126         {
1127           u_key.addr = m->local_addr;
1128           u_key.fib_index = m->fib_index;
1129           kv.key = u_key.as_u64;
1130           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1131             {
1132               user_index = value.value;
1133               u = pool_elt_at_index (tsm->users, user_index);
1134               if (u->nsessions)
1135                 {
1136                   head_index = u->sessions_per_user_list_head_index;
1137                   head = pool_elt_at_index (tsm->list_pool, head_index);
1138                   elt_index = head->next;
1139                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1140                   ses_index = elt->value;
1141                   while (ses_index != ~0)
1142                     {
1143                       s = pool_elt_at_index (tsm->sessions, ses_index);
1144                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1145                       ses_index = elt->value;
1146
1147                       if (snat_is_session_static (s))
1148                         continue;
1149
1150                       if (!addr_only && s->in2out.port != m->local_port)
1151                         continue;
1152
1153                       nat_free_session_data (sm, s,
1154                                              tsm - sm->per_thread_data, 0);
1155                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1156
1157                       if (!addr_only && !sm->endpoint_dependent)
1158                         break;
1159                     }
1160                 }
1161             }
1162         }
1163     }
1164   else
1165     {
1166       if (!m)
1167         {
1168           if (sw_if_index != ~0)
1169             return 0;
1170           else
1171             return VNET_API_ERROR_NO_SUCH_ENTRY;
1172         }
1173
1174       if (identity_nat)
1175         {
1176           if (vrf_id == ~0)
1177             vrf_id = sm->inside_vrf_id;
1178
1179           /* *INDENT-OFF* */
1180           pool_foreach (local, m->locals,
1181           ({
1182             if (local->vrf_id == vrf_id)
1183               find = local - m->locals;
1184           }));
1185           /* *INDENT-ON* */
1186           if (find == ~0)
1187             return VNET_API_ERROR_NO_SUCH_ENTRY;
1188
1189           local = pool_elt_at_index (m->locals, find);
1190           fib_index = local->fib_index;
1191           pool_put (m->locals, local);
1192         }
1193       else
1194         fib_index = m->fib_index;
1195
1196       /* Free external address port */
1197       if (!(addr_only || sm->static_mapping_only || out2in_only))
1198         {
1199           for (i = 0; i < vec_len (sm->addresses); i++)
1200             {
1201               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1202                 {
1203                   a = sm->addresses + i;
1204                   switch (proto)
1205                     {
1206 #define _(N, j, n, s) \
1207                     case NAT_PROTOCOL_##N: \
1208                       --a->busy_##n##_port_refcounts[e_port]; \
1209                       if (e_port > 1024) \
1210                         { \
1211                           a->busy_##n##_ports--; \
1212                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1213                         } \
1214                       break;
1215                       foreach_nat_protocol
1216 #undef _
1217                     default:
1218                       nat_elog_info ("unknown protocol");
1219                       return VNET_API_ERROR_INVALID_VALUE_2;
1220                     }
1221                   break;
1222                 }
1223             }
1224         }
1225
1226       if (sm->num_workers > 1)
1227         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1228       else
1229         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1230
1231       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1232       if (!out2in_only)
1233         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1234
1235       /* Delete session(s) for static mapping if exist */
1236       if (!(sm->static_mapping_only) ||
1237           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1238         {
1239           if (sm->endpoint_dependent)
1240             {
1241               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1242                                                    m->local_port, m->proto,
1243                                                    fib_index, addr_only,
1244                                                    e_addr, e_port);
1245             }
1246           else
1247             {
1248               u_key.addr = m->local_addr;
1249               u_key.fib_index = fib_index;
1250               kv.key = u_key.as_u64;
1251               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1252                                                 e_addr, e_port);
1253             }
1254         }
1255
1256       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1257       if (pool_elts (m->locals))
1258         return 0;
1259
1260       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1261       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1262
1263       vec_free (m->tag);
1264       vec_free (m->workers);
1265       /* Delete static mapping from pool */
1266       pool_put (sm->static_mappings, m);
1267     }
1268
1269   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1270     return 0;
1271
1272   /* Add/delete external address to FIB */
1273   /* *INDENT-OFF* */
1274   pool_foreach (interface, sm->interfaces,
1275   ({
1276     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1277       continue;
1278
1279     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1280     break;
1281   }));
1282   pool_foreach (interface, sm->output_feature_interfaces,
1283   ({
1284     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1285       continue;
1286
1287     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1288     break;
1289   }));
1290   /* *INDENT-ON* */
1291
1292   return 0;
1293 }
1294
1295 int
1296 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1297                                  nat_protocol_t proto,
1298                                  nat44_lb_addr_port_t * locals, u8 is_add,
1299                                  twice_nat_type_t twice_nat, u8 out2in_only,
1300                                  u8 * tag, u32 affinity)
1301 {
1302   snat_main_t *sm = &snat_main;
1303   snat_static_mapping_t *m;
1304   clib_bihash_kv_8_8_t kv, value;
1305   snat_address_t *a = 0;
1306   int i;
1307   nat44_lb_addr_port_t *local;
1308   snat_main_per_thread_data_t *tsm;
1309   snat_session_t *s;
1310   uword *bitmap = 0;
1311
1312   if (!sm->endpoint_dependent)
1313     return VNET_API_ERROR_FEATURE_DISABLED;
1314
1315   init_nat_k (&kv, e_addr, e_port, 0, proto);
1316   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1317     m = 0;
1318   else
1319     m = pool_elt_at_index (sm->static_mappings, value.value);
1320
1321   if (is_add)
1322     {
1323       if (m)
1324         return VNET_API_ERROR_VALUE_EXIST;
1325
1326       if (vec_len (locals) < 2)
1327         return VNET_API_ERROR_INVALID_VALUE;
1328
1329       /* Find external address in allocated addresses and reserve port for
1330          address and port pair mapping when dynamic translations enabled */
1331       if (!(sm->static_mapping_only || out2in_only))
1332         {
1333           for (i = 0; i < vec_len (sm->addresses); i++)
1334             {
1335               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1336                 {
1337                   a = sm->addresses + i;
1338                   /* External port must be unused */
1339                   switch (proto)
1340                     {
1341 #define _(N, j, n, s) \
1342                     case NAT_PROTOCOL_##N: \
1343                       if (a->busy_##n##_port_refcounts[e_port]) \
1344                         return VNET_API_ERROR_INVALID_VALUE; \
1345                       ++a->busy_##n##_port_refcounts[e_port]; \
1346                       if (e_port > 1024) \
1347                         { \
1348                           a->busy_##n##_ports++; \
1349                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1350                         } \
1351                       break;
1352                       foreach_nat_protocol
1353 #undef _
1354                     default:
1355                       nat_elog_info ("unknown protocol");
1356                       return VNET_API_ERROR_INVALID_VALUE_2;
1357                     }
1358                   break;
1359                 }
1360             }
1361           /* External address must be allocated */
1362           if (!a)
1363             return VNET_API_ERROR_NO_SUCH_ENTRY;
1364         }
1365
1366       pool_get (sm->static_mappings, m);
1367       clib_memset (m, 0, sizeof (*m));
1368       m->tag = vec_dup (tag);
1369       m->external_addr = e_addr;
1370       m->external_port = e_port;
1371       m->proto = proto;
1372       m->twice_nat = twice_nat;
1373       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1374       if (out2in_only)
1375         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1376       m->affinity = affinity;
1377
1378       if (affinity)
1379         m->affinity_per_service_list_head_index =
1380           nat_affinity_get_per_service_list_head_index ();
1381       else
1382         m->affinity_per_service_list_head_index = ~0;
1383
1384       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1385                    m - sm->static_mappings);
1386       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1387         {
1388           nat_elog_err ("static_mapping_by_external key add failed");
1389           return VNET_API_ERROR_UNSPECIFIED;
1390         }
1391
1392       for (i = 0; i < vec_len (locals); i++)
1393         {
1394           locals[i].fib_index =
1395             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1396                                                locals[i].vrf_id,
1397                                                nat_fib_src_low);
1398           if (!out2in_only)
1399             {
1400               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1401                            locals[i].fib_index, m->proto,
1402                            m - sm->static_mappings);
1403               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1404             }
1405           locals[i].prefix = (i == 0) ? locals[i].probability :
1406             (locals[i - 1].prefix + locals[i].probability);
1407           pool_get (m->locals, local);
1408           *local = locals[i];
1409           if (sm->num_workers > 1)
1410             {
1411               ip4_header_t ip = {
1412                 .src_address = locals[i].addr,
1413               };
1414               bitmap =
1415                 clib_bitmap_set (bitmap,
1416                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1417                                  1);
1418             }
1419         }
1420
1421       /* Assign workers */
1422       if (sm->num_workers > 1)
1423         {
1424           /* *INDENT-OFF* */
1425           clib_bitmap_foreach (i, bitmap,
1426             ({
1427                vec_add1(m->workers, i);
1428             }));
1429           /* *INDENT-ON* */
1430         }
1431     }
1432   else
1433     {
1434       if (!m)
1435         return VNET_API_ERROR_NO_SUCH_ENTRY;
1436
1437       if (!is_lb_static_mapping (m))
1438         return VNET_API_ERROR_INVALID_VALUE;
1439
1440       /* Free external address port */
1441       if (!(sm->static_mapping_only || out2in_only))
1442         {
1443           for (i = 0; i < vec_len (sm->addresses); i++)
1444             {
1445               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1446                 {
1447                   a = sm->addresses + i;
1448                   switch (proto)
1449                     {
1450 #define _(N, j, n, s) \
1451                     case NAT_PROTOCOL_##N: \
1452                       --a->busy_##n##_port_refcounts[e_port]; \
1453                       if (e_port > 1024) \
1454                         { \
1455                           a->busy_##n##_ports--; \
1456                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1457                         } \
1458                       break;
1459                       foreach_nat_protocol
1460 #undef _
1461                     default:
1462                       nat_elog_info ("unknown protocol");
1463                       return VNET_API_ERROR_INVALID_VALUE_2;
1464                     }
1465                   break;
1466                 }
1467             }
1468         }
1469
1470       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1471       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1472         {
1473           nat_elog_err ("static_mapping_by_external key del failed");
1474           return VNET_API_ERROR_UNSPECIFIED;
1475         }
1476
1477       /* *INDENT-OFF* */
1478       pool_foreach (local, m->locals,
1479       ({
1480           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1481                             nat_fib_src_low);
1482           if (!out2in_only)
1483             {
1484 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1485               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1486                 {
1487                   nat_elog_err ("static_mapping_by_local key del failed");
1488                   return VNET_API_ERROR_UNSPECIFIED;
1489                 }
1490             }
1491
1492           if (sm->num_workers > 1)
1493             {
1494               ip4_header_t ip = {
1495                 .src_address = local->addr,
1496               };
1497               tsm = vec_elt_at_index (sm->per_thread_data,
1498                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1499             }
1500           else
1501             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1502
1503           /* Delete sessions */
1504           pool_foreach (s, tsm->sessions, {
1505             if (!(is_lb_session (s)))
1506               continue;
1507
1508             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1509                 s->in2out.port != local->port)
1510               continue;
1511
1512             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1513             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1514           });
1515       }));
1516       /* *INDENT-ON* */
1517       if (m->affinity)
1518         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1519       pool_free (m->locals);
1520       vec_free (m->tag);
1521       vec_free (m->workers);
1522
1523       pool_put (sm->static_mappings, m);
1524     }
1525
1526   return 0;
1527 }
1528
1529 int
1530 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1531                                        ip4_address_t l_addr, u16 l_port,
1532                                        nat_protocol_t proto, u32 vrf_id,
1533                                        u8 probability, u8 is_add)
1534 {
1535   snat_main_t *sm = &snat_main;
1536   snat_static_mapping_t *m = 0;
1537   clib_bihash_kv_8_8_t kv, value;
1538   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1539   snat_main_per_thread_data_t *tsm;
1540   snat_session_t *s;
1541   u32 *locals = 0;
1542   uword *bitmap = 0;
1543   int i;
1544
1545   if (!sm->endpoint_dependent)
1546     return VNET_API_ERROR_FEATURE_DISABLED;
1547
1548   init_nat_k (&kv, e_addr, e_port, 0, proto);
1549   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1550     m = pool_elt_at_index (sm->static_mappings, value.value);
1551
1552   if (!m)
1553     return VNET_API_ERROR_NO_SUCH_ENTRY;
1554
1555   if (!is_lb_static_mapping (m))
1556     return VNET_API_ERROR_INVALID_VALUE;
1557
1558   /* *INDENT-OFF* */
1559   pool_foreach (local, m->locals,
1560   ({
1561     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1562         (local->vrf_id == vrf_id))
1563       {
1564         match_local = local;
1565         break;
1566       }
1567   }));
1568   /* *INDENT-ON* */
1569
1570   if (is_add)
1571     {
1572       if (match_local)
1573         return VNET_API_ERROR_VALUE_EXIST;
1574
1575       pool_get (m->locals, local);
1576       clib_memset (local, 0, sizeof (*local));
1577       local->addr.as_u32 = l_addr.as_u32;
1578       local->port = l_port;
1579       local->probability = probability;
1580       local->vrf_id = vrf_id;
1581       local->fib_index =
1582         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1583                                            nat_fib_src_low);
1584
1585       if (!is_out2in_only_static_mapping (m))
1586         {
1587           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto,
1588                        m - sm->static_mappings);
1589           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1590             nat_elog_err ("static_mapping_by_local key add failed");
1591         }
1592     }
1593   else
1594     {
1595       if (!match_local)
1596         return VNET_API_ERROR_NO_SUCH_ENTRY;
1597
1598       if (pool_elts (m->locals) < 3)
1599         return VNET_API_ERROR_UNSPECIFIED;
1600
1601       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1602                         nat_fib_src_low);
1603
1604       if (!is_out2in_only_static_mapping (m))
1605         {
1606           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1607           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1608             nat_elog_err ("static_mapping_by_local key del failed");
1609         }
1610
1611       if (sm->num_workers > 1)
1612         {
1613           ip4_header_t ip = {
1614             .src_address = local->addr,
1615           };
1616           tsm = vec_elt_at_index (sm->per_thread_data,
1617                                   sm->worker_in2out_cb (&ip, m->fib_index,
1618                                                         0));
1619         }
1620       else
1621         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1622
1623       /* Delete sessions */
1624       /* *INDENT-OFF* */
1625       pool_foreach (s, tsm->sessions, {
1626         if (!(is_lb_session (s)))
1627           continue;
1628
1629         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1630             s->in2out.port != match_local->port)
1631           continue;
1632
1633         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1634         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1635       });
1636       /* *INDENT-ON* */
1637
1638       pool_put (m->locals, match_local);
1639     }
1640
1641   vec_free (m->workers);
1642
1643   /* *INDENT-OFF* */
1644   pool_foreach (local, m->locals,
1645   ({
1646     vec_add1 (locals, local - m->locals);
1647     if (sm->num_workers > 1)
1648       {
1649         ip4_header_t ip;
1650         ip.src_address.as_u32 = local->addr.as_u32,
1651         bitmap = clib_bitmap_set (bitmap,
1652                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1653                                   1);
1654       }
1655   }));
1656   /* *INDENT-ON* */
1657
1658   ASSERT (vec_len (locals) > 1);
1659
1660   local = pool_elt_at_index (m->locals, locals[0]);
1661   local->prefix = local->probability;
1662   for (i = 1; i < vec_len (locals); i++)
1663     {
1664       local = pool_elt_at_index (m->locals, locals[i]);
1665       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1666       local->prefix = local->probability + prev_local->prefix;
1667     }
1668
1669   /* Assign workers */
1670   if (sm->num_workers > 1)
1671     {
1672       /* *INDENT-OFF* */
1673       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1674       /* *INDENT-ON* */
1675     }
1676
1677   return 0;
1678 }
1679
1680 int
1681 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1682                   u8 twice_nat)
1683 {
1684   snat_address_t *a = 0;
1685   snat_session_t *ses;
1686   u32 *ses_to_be_removed = 0, *ses_index;
1687   snat_main_per_thread_data_t *tsm;
1688   snat_static_mapping_t *m;
1689   snat_interface_t *interface;
1690   int i;
1691   snat_address_t *addresses =
1692     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1693
1694   /* Find SNAT address */
1695   for (i = 0; i < vec_len (addresses); i++)
1696     {
1697       if (addresses[i].addr.as_u32 == addr.as_u32)
1698         {
1699           a = addresses + i;
1700           break;
1701         }
1702     }
1703   if (!a)
1704     return VNET_API_ERROR_NO_SUCH_ENTRY;
1705
1706   if (delete_sm)
1707     {
1708       /* *INDENT-OFF* */
1709       pool_foreach (m, sm->static_mappings,
1710       ({
1711           if (m->external_addr.as_u32 == addr.as_u32)
1712             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1713                                             m->local_port, m->external_port,
1714                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1715                                             m->proto, 0, m->twice_nat,
1716                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1717       }));
1718       /* *INDENT-ON* */
1719     }
1720   else
1721     {
1722       /* Check if address is used in some static mapping */
1723       if (is_snat_address_used_in_static_mapping (sm, addr))
1724         {
1725           nat_elog_notice ("address used in static mapping");
1726           return VNET_API_ERROR_UNSPECIFIED;
1727         }
1728     }
1729
1730   if (a->fib_index != ~0)
1731     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1732
1733   /* Delete sessions using address */
1734   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1735     {
1736       /* *INDENT-OFF* */
1737       vec_foreach (tsm, sm->per_thread_data)
1738         {
1739           pool_foreach (ses, tsm->sessions, ({
1740             if (ses->out2in.addr.as_u32 == addr.as_u32)
1741               {
1742                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1743                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1744               }
1745           }));
1746
1747           if (sm->endpoint_dependent){
1748               vec_foreach (ses_index, ses_to_be_removed)
1749                 {
1750                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1751                   nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1752                 }
1753           }else{
1754               vec_foreach (ses_index, ses_to_be_removed)
1755                 {
1756                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1757                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1758                 }
1759           }
1760
1761           vec_free (ses_to_be_removed);
1762         }
1763       /* *INDENT-ON* */
1764     }
1765
1766 #define _(N, i, n, s) \
1767   vec_free (a->busy_##n##_ports_per_thread);
1768   foreach_nat_protocol
1769 #undef _
1770     if (twice_nat)
1771     {
1772       vec_del1 (sm->twice_nat_addresses, i);
1773       return 0;
1774     }
1775   else
1776     vec_del1 (sm->addresses, i);
1777
1778   /* Delete external address from FIB */
1779   /* *INDENT-OFF* */
1780   pool_foreach (interface, sm->interfaces,
1781   ({
1782     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1783       continue;
1784
1785     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1786     break;
1787   }));
1788   pool_foreach (interface, sm->output_feature_interfaces,
1789   ({
1790     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1791       continue;
1792
1793     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1794     break;
1795   }));
1796   /* *INDENT-ON* */
1797
1798   return 0;
1799 }
1800
1801 int
1802 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1803 {
1804   snat_main_t *sm = &snat_main;
1805   snat_interface_t *i;
1806   const char *feature_name, *del_feature_name;
1807   snat_address_t *ap;
1808   snat_static_mapping_t *m;
1809   snat_det_map_t *dm;
1810   nat_outside_fib_t *outside_fib;
1811   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1812                                                        sw_if_index);
1813
1814   if (sm->out2in_dpo && !is_inside)
1815     return VNET_API_ERROR_UNSUPPORTED;
1816
1817   /* *INDENT-OFF* */
1818   pool_foreach (i, sm->output_feature_interfaces,
1819   ({
1820     if (i->sw_if_index == sw_if_index)
1821       return VNET_API_ERROR_VALUE_EXIST;
1822   }));
1823   /* *INDENT-ON* */
1824
1825   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1826     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1827   else
1828     {
1829       if (sm->num_workers > 1 && !sm->deterministic)
1830         feature_name =
1831           is_inside ? "nat44-in2out-worker-handoff" :
1832           "nat44-out2in-worker-handoff";
1833       else if (sm->deterministic)
1834         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1835       else if (sm->endpoint_dependent)
1836         {
1837           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1838         }
1839       else
1840         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1841     }
1842
1843   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1844     sm->fq_in2out_index =
1845       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1846
1847   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1848     sm->fq_out2in_index =
1849       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1850
1851   if (!is_inside)
1852     {
1853       /* *INDENT-OFF* */
1854       vec_foreach (outside_fib, sm->outside_fibs)
1855         {
1856           if (outside_fib->fib_index == fib_index)
1857             {
1858               if (is_del)
1859                 {
1860                   outside_fib->refcount--;
1861                   if (!outside_fib->refcount)
1862                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1863                 }
1864               else
1865                 outside_fib->refcount++;
1866               goto feature_set;
1867             }
1868         }
1869       /* *INDENT-ON* */
1870       if (!is_del)
1871         {
1872           vec_add2 (sm->outside_fibs, outside_fib, 1);
1873           outside_fib->refcount = 1;
1874           outside_fib->fib_index = fib_index;
1875         }
1876     }
1877 feature_set:
1878   /* *INDENT-OFF* */
1879   pool_foreach (i, sm->interfaces,
1880   ({
1881     if (i->sw_if_index == sw_if_index)
1882       {
1883         if (is_del)
1884           {
1885             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1886               {
1887                 if (is_inside)
1888                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1889                 else
1890                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1891
1892                 if (sm->num_workers > 1 && !sm->deterministic)
1893                   {
1894                     del_feature_name = "nat44-handoff-classify";
1895                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1896                                                  "nat44-out2in-worker-handoff";
1897                   }
1898                 else if (sm->deterministic)
1899                   {
1900                     del_feature_name = "nat44-det-classify";
1901                     feature_name = !is_inside ?  "nat44-det-in2out" :
1902                                                  "nat44-det-out2in";
1903                   }
1904                 else if (sm->endpoint_dependent)
1905                   {
1906                     del_feature_name = "nat44-ed-classify";
1907                     feature_name = !is_inside ?  "nat-pre-in2out" :
1908                                                  "nat-pre-out2in";
1909                   }
1910                 else
1911                   {
1912                     del_feature_name = "nat44-classify";
1913                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1914                   }
1915
1916                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1917                 if (rv)
1918                   return rv;
1919                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1920                                              sw_if_index, 0, 0, 0);
1921                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1922                                              sw_if_index, 1, 0, 0);
1923                 if (!is_inside)
1924                   {
1925                     if (sm->endpoint_dependent)
1926                       vnet_feature_enable_disable ("ip4-local",
1927                                                    "nat44-ed-hairpinning",
1928                                                    sw_if_index, 1, 0, 0);
1929                     else if (!sm->deterministic)
1930                       vnet_feature_enable_disable ("ip4-local",
1931                                                    "nat44-hairpinning",
1932                                                    sw_if_index, 1, 0, 0);
1933                   }
1934               }
1935             else
1936               {
1937                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1938                 if (rv)
1939                   return rv;
1940                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1941                                              sw_if_index, 0, 0, 0);
1942                 pool_put (sm->interfaces, i);
1943                 if (is_inside)
1944                   {
1945                     if (sm->endpoint_dependent)
1946                       vnet_feature_enable_disable ("ip4-local",
1947                                                    "nat44-ed-hairpinning",
1948                                                    sw_if_index, 0, 0, 0);
1949                     else if (!sm->deterministic)
1950                       vnet_feature_enable_disable ("ip4-local",
1951                                                    "nat44-hairpinning",
1952                                                    sw_if_index, 0, 0, 0);
1953                   }
1954               }
1955           }
1956         else
1957           {
1958             if ((nat_interface_is_inside(i) && is_inside) ||
1959                 (nat_interface_is_outside(i) && !is_inside))
1960               return 0;
1961
1962             if (sm->num_workers > 1 && !sm->deterministic)
1963               {
1964                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1965                                                  "nat44-out2in-worker-handoff";
1966                 feature_name = "nat44-handoff-classify";
1967               }
1968             else if (sm->deterministic)
1969               {
1970                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1971                                                  "nat44-det-out2in";
1972                 feature_name = "nat44-det-classify";
1973               }
1974             else if (sm->endpoint_dependent)
1975               {
1976                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
1977                                                  "nat-pre-out2in";
1978
1979                 feature_name = "nat44-ed-classify";
1980               }
1981             else
1982               {
1983                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1984                 feature_name = "nat44-classify";
1985               }
1986
1987             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1988             if (rv)
1989               return rv;
1990             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1991                                          sw_if_index, 0, 0, 0);
1992             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1993                                          sw_if_index, 1, 0, 0);
1994             if (!is_inside)
1995               {
1996                 if (sm->endpoint_dependent)
1997                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1998                                                sw_if_index, 0, 0, 0);
1999                 else if (!sm->deterministic)
2000                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2001                                                sw_if_index, 0, 0, 0);
2002               }
2003             goto set_flags;
2004           }
2005
2006         goto fib;
2007       }
2008   }));
2009   /* *INDENT-ON* */
2010
2011   if (is_del)
2012     return VNET_API_ERROR_NO_SUCH_ENTRY;
2013
2014   pool_get (sm->interfaces, i);
2015   i->sw_if_index = sw_if_index;
2016   i->flags = 0;
2017   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2018                                0);
2019
2020   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2021   if (rv)
2022     return rv;
2023
2024   if (is_inside && !sm->out2in_dpo)
2025     {
2026       if (sm->endpoint_dependent)
2027         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2028                                      sw_if_index, 1, 0, 0);
2029       else if (!sm->deterministic)
2030         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2031                                      sw_if_index, 1, 0, 0);
2032     }
2033
2034 set_flags:
2035   if (is_inside)
2036     {
2037       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2038       return 0;
2039     }
2040   else
2041     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2042
2043   /* Add/delete external addresses to FIB */
2044 fib:
2045   /* *INDENT-OFF* */
2046   vec_foreach (ap, sm->addresses)
2047     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2048
2049   pool_foreach (m, sm->static_mappings,
2050   ({
2051     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2052       continue;
2053
2054     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2055   }));
2056
2057   pool_foreach (dm, sm->det_maps,
2058   ({
2059     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2060   }));
2061   /* *INDENT-ON* */
2062
2063   return 0;
2064 }
2065
2066 int
2067 snat_interface_add_del_output_feature (u32 sw_if_index,
2068                                        u8 is_inside, int is_del)
2069 {
2070   snat_main_t *sm = &snat_main;
2071   snat_interface_t *i;
2072   snat_address_t *ap;
2073   snat_static_mapping_t *m;
2074   nat_outside_fib_t *outside_fib;
2075   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2076                                                        sw_if_index);
2077
2078
2079   if (sm->deterministic ||
2080       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2081     return VNET_API_ERROR_UNSUPPORTED;
2082
2083   /* *INDENT-OFF* */
2084   pool_foreach (i, sm->interfaces,
2085   ({
2086     if (i->sw_if_index == sw_if_index)
2087       return VNET_API_ERROR_VALUE_EXIST;
2088   }));
2089   /* *INDENT-ON* */
2090
2091   if (!is_inside)
2092     {
2093       /* *INDENT-OFF* */
2094       vec_foreach (outside_fib, sm->outside_fibs)
2095         {
2096           if (outside_fib->fib_index == fib_index)
2097             {
2098               if (is_del)
2099                 {
2100                   outside_fib->refcount--;
2101                   if (!outside_fib->refcount)
2102                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2103                 }
2104               else
2105                 outside_fib->refcount++;
2106               goto feature_set;
2107             }
2108         }
2109       /* *INDENT-ON* */
2110       if (!is_del)
2111         {
2112           vec_add2 (sm->outside_fibs, outside_fib, 1);
2113           outside_fib->refcount = 1;
2114           outside_fib->fib_index = fib_index;
2115         }
2116     }
2117
2118 feature_set:
2119   if (is_inside)
2120     {
2121       if (sm->endpoint_dependent)
2122         {
2123           int rv =
2124             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2125           if (rv)
2126             return rv;
2127           rv =
2128             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2129                                                             !is_del);
2130           if (rv)
2131             return rv;
2132           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2133                                        sw_if_index, !is_del, 0, 0);
2134           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2135                                        sw_if_index, !is_del, 0, 0);
2136         }
2137       else
2138         {
2139           int rv =
2140             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2141           if (rv)
2142             return rv;
2143           rv =
2144             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2145                                                             !is_del);
2146           if (rv)
2147             return rv;
2148           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2149                                        sw_if_index, !is_del, 0, 0);
2150           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2151                                        sw_if_index, !is_del, 0, 0);
2152         }
2153       goto fq;
2154     }
2155
2156   if (sm->num_workers > 1)
2157     {
2158       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2159       if (rv)
2160         return rv;
2161       rv =
2162         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2163       if (rv)
2164         return rv;
2165       vnet_feature_enable_disable ("ip4-unicast",
2166                                    "nat44-out2in-worker-handoff",
2167                                    sw_if_index, !is_del, 0, 0);
2168       vnet_feature_enable_disable ("ip4-output",
2169                                    "nat44-in2out-output-worker-handoff",
2170                                    sw_if_index, !is_del, 0, 0);
2171     }
2172   else
2173     {
2174       if (sm->endpoint_dependent)
2175         {
2176           int rv =
2177             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2178           if (rv)
2179             return rv;
2180           rv =
2181             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2182                                                             !is_del);
2183           if (rv)
2184             return rv;
2185           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2186                                        sw_if_index, !is_del, 0, 0);
2187           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2188                                        sw_if_index, !is_del, 0, 0);
2189         }
2190       else
2191         {
2192           int rv =
2193             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2194           if (rv)
2195             return rv;
2196           rv =
2197             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2198                                                             !is_del);
2199           if (rv)
2200             return rv;
2201           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2202                                        sw_if_index, !is_del, 0, 0);
2203           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2204                                        sw_if_index, !is_del, 0, 0);
2205         }
2206     }
2207
2208 fq:
2209   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2210     sm->fq_in2out_output_index =
2211       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2212
2213   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2214     sm->fq_out2in_index =
2215       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2216
2217   /* *INDENT-OFF* */
2218   pool_foreach (i, sm->output_feature_interfaces,
2219   ({
2220     if (i->sw_if_index == sw_if_index)
2221       {
2222         if (is_del)
2223           pool_put (sm->output_feature_interfaces, i);
2224         else
2225           return VNET_API_ERROR_VALUE_EXIST;
2226
2227         goto fib;
2228       }
2229   }));
2230   /* *INDENT-ON* */
2231
2232   if (is_del)
2233     return VNET_API_ERROR_NO_SUCH_ENTRY;
2234
2235   pool_get (sm->output_feature_interfaces, i);
2236   i->sw_if_index = sw_if_index;
2237   i->flags = 0;
2238   if (is_inside)
2239     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2240   else
2241     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2242
2243   /* Add/delete external addresses to FIB */
2244 fib:
2245   if (is_inside)
2246     return 0;
2247
2248   /* *INDENT-OFF* */
2249   vec_foreach (ap, sm->addresses)
2250     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2251
2252   pool_foreach (m, sm->static_mappings,
2253   ({
2254     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2255       continue;
2256
2257     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2258   }));
2259   /* *INDENT-ON* */
2260
2261   return 0;
2262 }
2263
2264 int
2265 snat_set_workers (uword * bitmap)
2266 {
2267   snat_main_t *sm = &snat_main;
2268   int i, j = 0;
2269
2270   if (sm->num_workers < 2)
2271     return VNET_API_ERROR_FEATURE_DISABLED;
2272
2273   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2274     return VNET_API_ERROR_INVALID_WORKER;
2275
2276   vec_free (sm->workers);
2277   /* *INDENT-OFF* */
2278   clib_bitmap_foreach (i, bitmap,
2279     ({
2280       vec_add1(sm->workers, i);
2281       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2282       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2283       j++;
2284     }));
2285   /* *INDENT-ON* */
2286
2287   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2288
2289   return 0;
2290 }
2291
2292 static void
2293 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2294                          u32 old_fib_index)
2295 {
2296   snat_main_t *sm = &snat_main;
2297   nat_outside_fib_t *outside_fib;
2298   snat_interface_t *i;
2299   u8 is_add = 1;
2300   u8 match = 0;
2301
2302   if (new_fib_index == old_fib_index)
2303     return;
2304
2305   if (!vec_len (sm->outside_fibs))
2306     return;
2307
2308   /* *INDENT-OFF* */
2309   pool_foreach (i, sm->interfaces,
2310     ({
2311       if (i->sw_if_index == sw_if_index)
2312         {
2313           if (!(nat_interface_is_outside (i)))
2314             return;
2315           match = 1;
2316         }
2317     }));
2318
2319   pool_foreach (i, sm->output_feature_interfaces,
2320     ({
2321       if (i->sw_if_index == sw_if_index)
2322         {
2323           if (!(nat_interface_is_outside (i)))
2324             return;
2325           match = 1;
2326         }
2327     }));
2328   /* *INDENT-ON* */
2329
2330   if (!match)
2331     return;
2332
2333   vec_foreach (outside_fib, sm->outside_fibs)
2334   {
2335     if (outside_fib->fib_index == old_fib_index)
2336       {
2337         outside_fib->refcount--;
2338         if (!outside_fib->refcount)
2339           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2340         break;
2341       }
2342   }
2343
2344   vec_foreach (outside_fib, sm->outside_fibs)
2345   {
2346     if (outside_fib->fib_index == new_fib_index)
2347       {
2348         outside_fib->refcount++;
2349         is_add = 0;
2350         break;
2351       }
2352   }
2353
2354   if (is_add)
2355     {
2356       vec_add2 (sm->outside_fibs, outside_fib, 1);
2357       outside_fib->refcount = 1;
2358       outside_fib->fib_index = new_fib_index;
2359     }
2360 }
2361
2362 static void
2363 snat_ip4_table_bind (ip4_main_t * im,
2364                      uword opaque,
2365                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2366 {
2367   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2368 }
2369
2370 static void
2371 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2372                                        uword opaque,
2373                                        u32 sw_if_index,
2374                                        ip4_address_t * address,
2375                                        u32 address_length,
2376                                        u32 if_address_index, u32 is_delete);
2377
2378 static void
2379 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2380                                  uword opaque,
2381                                  u32 sw_if_index,
2382                                  ip4_address_t * address,
2383                                  u32 address_length,
2384                                  u32 if_address_index, u32 is_delete);
2385
2386 static int
2387 nat_alloc_addr_and_port_default (snat_address_t * addresses, u32 fib_index,
2388                                  u32 thread_index, nat_protocol_t proto,
2389                                  ip4_address_t * addr, u16 * port,
2390                                  u16 port_per_thread, u32 snat_thread_index);
2391
2392 void
2393 test_key_calc_split ()
2394 {
2395   ip4_address_t l_addr;
2396   l_addr.as_u8[0] = 1;
2397   l_addr.as_u8[1] = 1;
2398   l_addr.as_u8[2] = 1;
2399   l_addr.as_u8[3] = 1;
2400   ip4_address_t r_addr;
2401   r_addr.as_u8[0] = 2;
2402   r_addr.as_u8[1] = 2;
2403   r_addr.as_u8[2] = 2;
2404   r_addr.as_u8[3] = 2;
2405   u16 l_port = 40001;
2406   u16 r_port = 40301;
2407   u8 proto = 9;
2408   u32 fib_index = 9000001;
2409   u32 thread_index = 3000000001;
2410   u32 session_index = 3000000221;
2411   clib_bihash_kv_16_8_t kv;
2412   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2413               thread_index, session_index);
2414   ip4_address_t l_addr2;
2415   ip4_address_t r_addr2;
2416   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2417   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2418   u16 l_port2 = 0;
2419   u16 r_port2 = 0;
2420   u8 proto2 = 0;
2421   u32 fib_index2 = 0;
2422   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2423                &r_port2);
2424   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2425   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2426   ASSERT (l_port == l_port2);
2427   ASSERT (r_port == r_port2);
2428   ASSERT (proto == proto2);
2429   ASSERT (fib_index == fib_index2);
2430   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2431   ASSERT (session_index == ed_value_get_session_index (&kv));
2432
2433   fib_index = 7001;
2434   proto = 5;
2435   nat_protocol_t proto3 = ~0;
2436   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2437   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2438   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2439   ASSERT (l_port == l_port2);
2440   ASSERT (proto == proto3);
2441   ASSERT (fib_index == fib_index2);
2442 }
2443
2444 static clib_error_t *
2445 snat_init (vlib_main_t * vm)
2446 {
2447   snat_main_t *sm = &snat_main;
2448   clib_error_t *error = 0;
2449   ip4_main_t *im = &ip4_main;
2450   ip_lookup_main_t *lm = &im->lookup_main;
2451   uword *p;
2452   vlib_thread_registration_t *tr;
2453   vlib_thread_main_t *tm = vlib_get_thread_main ();
2454   uword *bitmap = 0;
2455   u32 i;
2456   ip4_add_del_interface_address_callback_t cb4;
2457   vlib_node_t *node;
2458
2459   sm->vnet_main = vnet_get_main ();
2460   sm->ip4_main = im;
2461   sm->ip4_lookup_main = lm;
2462   sm->api_main = vlibapi_get_main ();
2463   sm->first_worker_index = 0;
2464   sm->num_workers = 0;
2465   sm->workers = 0;
2466   sm->port_per_thread = 0xffff - 1024;
2467   sm->fq_in2out_index = ~0;
2468   sm->fq_in2out_output_index = ~0;
2469   sm->fq_out2in_index = ~0;
2470
2471   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2472   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2473   sm->forwarding_enabled = 0;
2474   sm->log_class = vlib_log_register_class ("nat", 0);
2475   sm->log_level = SNAT_LOG_ERROR;
2476   sm->mss_clamping = 0;
2477
2478   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2479   sm->error_node_index = node->index;
2480
2481   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2482   sm->pre_in2out_node_index = node->index;
2483   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2484   sm->pre_out2in_node_index = node->index;
2485
2486   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2487   sm->pre_in2out_node_index = node->index;
2488
2489   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2490   sm->pre_out2in_node_index = node->index;
2491
2492   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2493   sm->in2out_node_index = node->index;
2494   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2495   sm->in2out_output_node_index = node->index;
2496   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2497   sm->in2out_fast_node_index = node->index;
2498   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2499   sm->in2out_slowpath_node_index = node->index;
2500   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2501   sm->in2out_slowpath_output_node_index = node->index;
2502
2503   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2504   sm->ed_in2out_node_index = node->index;
2505   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2506   sm->ed_in2out_slowpath_node_index = node->index;
2507
2508   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2509   sm->out2in_node_index = node->index;
2510   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2511   sm->out2in_fast_node_index = node->index;
2512
2513   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2514   sm->ed_out2in_node_index = node->index;
2515   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2516   sm->ed_out2in_slowpath_node_index = node->index;
2517
2518   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2519   sm->det_in2out_node_index = node->index;
2520   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2521   sm->det_out2in_node_index = node->index;
2522
2523   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2524   sm->hairpinning_node_index = node->index;
2525   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2526   sm->hairpin_dst_node_index = node->index;
2527   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2528   sm->hairpin_src_node_index = node->index;
2529   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2530   sm->ed_hairpinning_node_index = node->index;
2531   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2532   sm->ed_hairpin_dst_node_index = node->index;
2533   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2534   sm->ed_hairpin_src_node_index = node->index;
2535
2536   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2537   if (p)
2538     {
2539       tr = (vlib_thread_registration_t *) p[0];
2540       if (tr)
2541         {
2542           sm->num_workers = tr->count;
2543           sm->first_worker_index = tr->first_index;
2544         }
2545     }
2546
2547   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2548
2549   /* Use all available workers by default */
2550   if (sm->num_workers > 1)
2551     {
2552       for (i = 0; i < sm->num_workers; i++)
2553         bitmap = clib_bitmap_set (bitmap, i, 1);
2554       snat_set_workers (bitmap);
2555       clib_bitmap_free (bitmap);
2556     }
2557   else
2558     {
2559       sm->per_thread_data[0].snat_thread_index = 0;
2560     }
2561
2562   error = snat_api_init (vm, sm);
2563   if (error)
2564     return error;
2565
2566   /* Set up the interface address add/del callback */
2567   cb4.function = snat_ip4_add_del_interface_address_cb;
2568   cb4.function_opaque = 0;
2569
2570   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2571
2572   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2573   cb4.function_opaque = 0;
2574
2575   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2576
2577   nat_dpo_module_init ();
2578
2579   /* Init counters */
2580   sm->total_users.name = "total-users";
2581   sm->total_users.stat_segment_name = "/nat44/total-users";
2582   vlib_validate_simple_counter (&sm->total_users, 0);
2583   vlib_zero_simple_counter (&sm->total_users, 0);
2584   sm->total_sessions.name = "total-sessions";
2585   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2586   vlib_validate_simple_counter (&sm->total_sessions, 0);
2587   vlib_zero_simple_counter (&sm->total_sessions, 0);
2588   sm->user_limit_reached.name = "user-limit-reached";
2589   sm->user_limit_reached.stat_segment_name = "/nat44/user-limit-reached";
2590   vlib_validate_simple_counter (&sm->user_limit_reached, 0);
2591   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2592
2593   /* Init IPFIX logging */
2594   snat_ipfix_logging_init (vm);
2595
2596   /* Init NAT64 */
2597   error = nat64_init (vm);
2598   if (error)
2599     return error;
2600
2601   nat66_init (vm);
2602
2603   ip4_table_bind_callback_t cbt4 = {
2604     .function = snat_ip4_table_bind,
2605   };
2606   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2607
2608   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2609                                         FIB_SOURCE_PRIORITY_HI,
2610                                         FIB_SOURCE_BH_SIMPLE);
2611   nat_fib_src_low = fib_source_allocate ("nat-low",
2612                                          FIB_SOURCE_PRIORITY_LOW,
2613                                          FIB_SOURCE_BH_SIMPLE);
2614
2615   test_key_calc_split ();
2616   return error;
2617 }
2618
2619 VLIB_INIT_FUNCTION (snat_init);
2620
2621 void
2622 snat_free_outside_address_and_port (snat_address_t * addresses,
2623                                     u32 thread_index,
2624                                     ip4_address_t * addr,
2625                                     u16 port, nat_protocol_t protocol)
2626 {
2627   snat_address_t *a;
2628   u32 address_index;
2629   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2630
2631   for (address_index = 0; address_index < vec_len (addresses);
2632        address_index++)
2633     {
2634       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2635         break;
2636     }
2637
2638   ASSERT (address_index < vec_len (addresses));
2639
2640   a = addresses + address_index;
2641
2642   switch (protocol)
2643     {
2644 #define _(N, i, n, s) \
2645     case NAT_PROTOCOL_##N: \
2646       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2647       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2648       a->busy_##n##_ports--; \
2649       a->busy_##n##_ports_per_thread[thread_index]--; \
2650       break;
2651       foreach_nat_protocol
2652 #undef _
2653     default:
2654       nat_elog_info ("unknown protocol");
2655       return;
2656     }
2657 }
2658
2659 static int
2660 nat_set_outside_address_and_port (snat_address_t * addresses,
2661                                   u32 thread_index, ip4_address_t addr,
2662                                   u16 port, nat_protocol_t protocol)
2663 {
2664   snat_address_t *a = 0;
2665   u32 address_index;
2666   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2667
2668   for (address_index = 0; address_index < vec_len (addresses);
2669        address_index++)
2670     {
2671       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2672         continue;
2673
2674       a = addresses + address_index;
2675       switch (protocol)
2676         {
2677 #define _(N, j, n, s) \
2678         case NAT_PROTOCOL_##N: \
2679           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2680             return VNET_API_ERROR_INSTANCE_IN_USE; \
2681           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2682           a->busy_##n##_ports_per_thread[thread_index]++; \
2683           a->busy_##n##_ports++; \
2684           return 0;
2685           foreach_nat_protocol
2686 #undef _
2687         default:
2688           nat_elog_info ("unknown protocol");
2689           return 1;
2690         }
2691     }
2692
2693   return VNET_API_ERROR_NO_SUCH_ENTRY;
2694 }
2695
2696 int
2697 snat_static_mapping_match (snat_main_t * sm,
2698                            ip4_address_t match_addr,
2699                            u16 match_port,
2700                            u32 match_fib_index,
2701                            nat_protocol_t match_protocol,
2702                            ip4_address_t * mapping_addr,
2703                            u16 * mapping_port,
2704                            u32 * mapping_fib_index,
2705                            u8 by_external,
2706                            u8 * is_addr_only,
2707                            twice_nat_type_t * twice_nat,
2708                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2709                            u8 * is_identity_nat)
2710 {
2711   clib_bihash_kv_8_8_t kv, value;
2712   snat_static_mapping_t *m;
2713   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2714   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2715   u8 backend_index;
2716   nat44_lb_addr_port_t *local;
2717
2718   if (by_external)
2719     {
2720       mapping_hash = &sm->static_mapping_by_external;
2721       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2722       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2723         {
2724           /* Try address only mapping */
2725           init_nat_k (&kv, match_addr, 0, 0, 0);
2726           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2727             return 1;
2728         }
2729
2730     }
2731   else
2732     {
2733       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2734                   match_protocol);
2735       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2736         {
2737           /* Try address only mapping */
2738           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2739           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2740             return 1;
2741         }
2742
2743     }
2744
2745   m = pool_elt_at_index (sm->static_mappings, value.value);
2746
2747   if (by_external)
2748     {
2749       if (is_lb_static_mapping (m))
2750         {
2751           if (PREDICT_FALSE (lb != 0))
2752             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2753           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2754                                                           match_addr,
2755                                                           match_protocol,
2756                                                           match_port,
2757                                                           &backend_index))
2758             {
2759               local = pool_elt_at_index (m->locals, backend_index);
2760               *mapping_addr = local->addr;
2761               *mapping_port = local->port;
2762               *mapping_fib_index = local->fib_index;
2763               goto end;
2764             }
2765           // pick locals matching this worker
2766           if (PREDICT_FALSE (sm->num_workers > 1))
2767             {
2768               u32 thread_index = vlib_get_thread_index ();
2769               /* *INDENT-OFF* */
2770               pool_foreach_index (i, m->locals,
2771               ({
2772                 local = pool_elt_at_index (m->locals, i);
2773
2774                 ip4_header_t ip = {
2775                   .src_address = local->addr,
2776                 };
2777
2778                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2779                     thread_index)
2780                   {
2781                     vec_add1 (tmp, i);
2782                   }
2783               }));
2784               /* *INDENT-ON* */
2785               ASSERT (vec_len (tmp) != 0);
2786             }
2787           else
2788             {
2789               /* *INDENT-OFF* */
2790               pool_foreach_index (i, m->locals,
2791               ({
2792                 vec_add1 (tmp, i);
2793               }));
2794               /* *INDENT-ON* */
2795             }
2796           hi = vec_len (tmp) - 1;
2797           local = pool_elt_at_index (m->locals, tmp[hi]);
2798           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2799           while (lo < hi)
2800             {
2801               mid = ((hi - lo) >> 1) + lo;
2802               local = pool_elt_at_index (m->locals, tmp[mid]);
2803               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2804             }
2805           local = pool_elt_at_index (m->locals, tmp[lo]);
2806           if (!(local->prefix >= rand))
2807             return 1;
2808           *mapping_addr = local->addr;
2809           *mapping_port = local->port;
2810           *mapping_fib_index = local->fib_index;
2811           if (m->affinity)
2812             {
2813               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2814                                                 match_protocol, match_port,
2815                                                 tmp[lo], m->affinity,
2816                                                 m->affinity_per_service_list_head_index))
2817                 nat_elog_info ("create affinity record failed");
2818             }
2819           vec_free (tmp);
2820         }
2821       else
2822         {
2823           if (PREDICT_FALSE (lb != 0))
2824             *lb = NO_LB_NAT;
2825           *mapping_fib_index = m->fib_index;
2826           *mapping_addr = m->local_addr;
2827           /* Address only mapping doesn't change port */
2828           *mapping_port = is_addr_only_static_mapping (m) ? match_port
2829             : m->local_port;
2830         }
2831     }
2832   else
2833     {
2834       *mapping_addr = m->external_addr;
2835       /* Address only mapping doesn't change port */
2836       *mapping_port = is_addr_only_static_mapping (m) ? match_port
2837         : m->external_port;
2838       *mapping_fib_index = sm->outside_fib_index;
2839     }
2840
2841 end:
2842   if (PREDICT_FALSE (is_addr_only != 0))
2843     *is_addr_only = is_addr_only_static_mapping (m);
2844
2845   if (PREDICT_FALSE (twice_nat != 0))
2846     *twice_nat = m->twice_nat;
2847
2848   if (PREDICT_FALSE (is_identity_nat != 0))
2849     *is_identity_nat = is_identity_static_mapping (m);
2850
2851   return 0;
2852 }
2853
2854 int
2855 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2856                                      u32 fib_index,
2857                                      u32 thread_index,
2858                                      nat_protocol_t proto,
2859                                      ip4_address_t * addr,
2860                                      u16 * port,
2861                                      u16 port_per_thread,
2862                                      u32 snat_thread_index)
2863 {
2864   snat_main_t *sm = &snat_main;
2865
2866   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, proto,
2867                                   addr, port, port_per_thread,
2868                                   snat_thread_index);
2869 }
2870
2871 static int
2872 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2873                                  u32 fib_index,
2874                                  u32 thread_index,
2875                                  nat_protocol_t proto,
2876                                  ip4_address_t * addr,
2877                                  u16 * port,
2878                                  u16 port_per_thread, u32 snat_thread_index)
2879 {
2880   int i;
2881   snat_address_t *a, *ga = 0;
2882   u32 portnum;
2883
2884   for (i = 0; i < vec_len (addresses); i++)
2885     {
2886       a = addresses + i;
2887       switch (proto)
2888         {
2889 #define _(N, j, n, s) \
2890         case NAT_PROTOCOL_##N: \
2891           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2892             { \
2893               if (a->fib_index == fib_index) \
2894                 { \
2895                   while (1) \
2896                     { \
2897                       portnum = (port_per_thread * \
2898                         snat_thread_index) + \
2899                         snat_random_port(1, port_per_thread) + 1024; \
2900                       if (a->busy_##n##_port_refcounts[portnum]) \
2901                         continue; \
2902                       --a->busy_##n##_port_refcounts[portnum]; \
2903                       a->busy_##n##_ports_per_thread[thread_index]++; \
2904                       a->busy_##n##_ports++; \
2905                       *addr = a->addr; \
2906                       *port = clib_host_to_net_u16(portnum); \
2907                       return 0; \
2908                     } \
2909                 } \
2910               else if (a->fib_index == ~0) \
2911                 { \
2912                   ga = a; \
2913                 } \
2914             } \
2915           break;
2916           foreach_nat_protocol
2917 #undef _
2918         default:
2919           nat_elog_info ("unknown protocol");
2920           return 1;
2921         }
2922
2923     }
2924
2925   if (ga)
2926     {
2927       a = ga;
2928       switch (proto)
2929         {
2930 #define _(N, j, n, s) \
2931         case NAT_PROTOCOL_##N: \
2932           while (1) \
2933             { \
2934               portnum = (port_per_thread * \
2935                 snat_thread_index) + \
2936                 snat_random_port(1, port_per_thread) + 1024; \
2937               if (a->busy_##n##_port_refcounts[portnum]) \
2938                 continue; \
2939               ++a->busy_##n##_port_refcounts[portnum]; \
2940               a->busy_##n##_ports_per_thread[thread_index]++; \
2941               a->busy_##n##_ports++; \
2942               *addr = a->addr; \
2943               *port = clib_host_to_net_u16(portnum); \
2944               return 0; \
2945             }
2946           break;
2947           foreach_nat_protocol
2948 #undef _
2949         default:
2950           nat_elog_info ("unknown protocol");
2951           return 1;
2952         }
2953     }
2954
2955   /* Totally out of translations to use... */
2956   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2957   return 1;
2958 }
2959
2960 static int
2961 nat_alloc_addr_and_port_mape (snat_address_t * addresses, u32 fib_index,
2962                               u32 thread_index, nat_protocol_t proto,
2963                               ip4_address_t * addr, u16 * port,
2964                               u16 port_per_thread, u32 snat_thread_index)
2965 {
2966   snat_main_t *sm = &snat_main;
2967   snat_address_t *a = addresses;
2968   u16 m, ports, portnum, A, j;
2969   m = 16 - (sm->psid_offset + sm->psid_length);
2970   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2971
2972   if (!vec_len (addresses))
2973     goto exhausted;
2974
2975   switch (proto)
2976     {
2977 #define _(N, i, n, s) \
2978     case NAT_PROTOCOL_##N: \
2979       if (a->busy_##n##_ports < ports) \
2980         { \
2981           while (1) \
2982             { \
2983               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2984               j = snat_random_port(0, pow2_mask(m)); \
2985               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2986               if (a->busy_##n##_port_refcounts[portnum]) \
2987                 continue; \
2988               ++a->busy_##n##_port_refcounts[portnum]; \
2989               a->busy_##n##_ports++; \
2990               *addr = a->addr; \
2991               *port = clib_host_to_net_u16 (portnum); \
2992               return 0; \
2993             } \
2994         } \
2995       break;
2996       foreach_nat_protocol
2997 #undef _
2998     default:
2999       nat_elog_info ("unknown protocol");
3000       return 1;
3001     }
3002
3003 exhausted:
3004   /* Totally out of translations to use... */
3005   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3006   return 1;
3007 }
3008
3009 static int
3010 nat_alloc_addr_and_port_range (snat_address_t * addresses, u32 fib_index,
3011                                u32 thread_index, nat_protocol_t proto,
3012                                ip4_address_t * addr, u16 * port,
3013                                u16 port_per_thread, u32 snat_thread_index)
3014 {
3015   snat_main_t *sm = &snat_main;
3016   snat_address_t *a = addresses;
3017   u16 portnum, ports;
3018
3019   ports = sm->end_port - sm->start_port + 1;
3020
3021   if (!vec_len (addresses))
3022     goto exhausted;
3023
3024   switch (proto)
3025     {
3026 #define _(N, i, n, s) \
3027     case NAT_PROTOCOL_##N: \
3028       if (a->busy_##n##_ports < ports) \
3029         { \
3030           while (1) \
3031             { \
3032               portnum = snat_random_port(sm->start_port, sm->end_port); \
3033               if (a->busy_##n##_port_refcounts[portnum]) \
3034                 continue; \
3035               ++a->busy_##n##_port_refcounts[portnum]; \
3036               a->busy_##n##_ports++; \
3037               *addr = a->addr; \
3038               *port = clib_host_to_net_u16 (portnum); \
3039               return 0; \
3040             } \
3041         } \
3042       break;
3043       foreach_nat_protocol
3044 #undef _
3045     default:
3046       nat_elog_info ("unknown protocol");
3047       return 1;
3048     }
3049
3050 exhausted:
3051   /* Totally out of translations to use... */
3052   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3053   return 1;
3054 }
3055
3056 void
3057 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3058 {
3059   dpo_id_t dpo_v4 = DPO_INVALID;
3060   fib_prefix_t pfx = {
3061     .fp_proto = FIB_PROTOCOL_IP4,
3062     .fp_len = 32,
3063     .fp_addr.ip4.as_u32 = addr.as_u32,
3064   };
3065
3066   if (is_add)
3067     {
3068       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3069       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3070                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3071       dpo_reset (&dpo_v4);
3072     }
3073   else
3074     {
3075       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3076     }
3077 }
3078
3079 u8 *
3080 format_session_kvp (u8 * s, va_list * args)
3081 {
3082   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3083
3084   s = format (s, "%U session-index %llu", format_snat_key, v->key, v->value);
3085
3086   return s;
3087 }
3088
3089 u8 *
3090 format_static_mapping_kvp (u8 * s, va_list * args)
3091 {
3092   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3093
3094   s = format (s, "%U static-mapping-index %llu",
3095               format_snat_key, v->key, v->value);
3096
3097   return s;
3098 }
3099
3100 u8 *
3101 format_user_kvp (u8 * s, va_list * args)
3102 {
3103   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3104   snat_user_key_t k;
3105
3106   k.as_u64 = v->key;
3107
3108   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3109               k.fib_index, v->value);
3110
3111   return s;
3112 }
3113
3114 u8 *
3115 format_ed_session_kvp (u8 * s, va_list * args)
3116 {
3117   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3118
3119   u8 proto;
3120   u16 r_port, l_port;
3121   ip4_address_t l_addr, r_addr;
3122   u32 fib_index;
3123
3124   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
3125   s =
3126     format (s,
3127             "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
3128             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
3129             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
3130             format_ip_protocol, proto, fib_index,
3131             ed_value_get_session_index (v), ed_value_get_thread_index (v));
3132
3133   return s;
3134 }
3135
3136 static u32
3137 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3138                            u8 is_output)
3139 {
3140   snat_main_t *sm = &snat_main;
3141   u32 next_worker_index = 0;
3142   u32 hash;
3143
3144   next_worker_index = sm->first_worker_index;
3145   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3146     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3147
3148   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3149     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3150   else
3151     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3152
3153   return next_worker_index;
3154 }
3155
3156 static u32
3157 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3158                            u32 rx_fib_index0, u8 is_output)
3159 {
3160   snat_main_t *sm = &snat_main;
3161   udp_header_t *udp;
3162   u16 port;
3163   clib_bihash_kv_8_8_t kv, value;
3164   snat_static_mapping_t *m;
3165   u32 proto;
3166   u32 next_worker_index = 0;
3167
3168   /* first try static mappings without port */
3169   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3170     {
3171       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
3172       if (!clib_bihash_search_8_8
3173           (&sm->static_mapping_by_external, &kv, &value))
3174         {
3175           m = pool_elt_at_index (sm->static_mappings, value.value);
3176           return m->workers[0];
3177         }
3178     }
3179
3180   proto = ip_proto_to_nat_proto (ip0->protocol);
3181   udp = ip4_next_header (ip0);
3182   port = udp->dst_port;
3183
3184   /* unknown protocol */
3185   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3186     {
3187       /* use current thread */
3188       return vlib_get_thread_index ();
3189     }
3190
3191   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3192     {
3193       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3194       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3195       if (!icmp_type_is_error_message
3196           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3197         port = vnet_buffer (b)->ip.reass.l4_src_port;
3198       else
3199         {
3200           /* if error message, then it's not fragmented and we can access it */
3201           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3202           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3203           void *l4_header = ip4_next_header (inner_ip);
3204           switch (proto)
3205             {
3206             case NAT_PROTOCOL_ICMP:
3207               icmp = (icmp46_header_t *) l4_header;
3208               echo = (icmp_echo_header_t *) (icmp + 1);
3209               port = echo->identifier;
3210               break;
3211             case NAT_PROTOCOL_UDP:
3212             case NAT_PROTOCOL_TCP:
3213               port = ((tcp_udp_header_t *) l4_header)->src_port;
3214               break;
3215             default:
3216               return vlib_get_thread_index ();
3217             }
3218         }
3219     }
3220
3221   /* try static mappings with port */
3222   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3223     {
3224       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
3225       if (!clib_bihash_search_8_8
3226           (&sm->static_mapping_by_external, &kv, &value))
3227         {
3228           m = pool_elt_at_index (sm->static_mappings, value.value);
3229           return m->workers[0];
3230         }
3231     }
3232
3233   /* worker by outside port */
3234   next_worker_index = sm->first_worker_index;
3235   next_worker_index +=
3236     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3237   return next_worker_index;
3238 }
3239
3240 static u32
3241 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3242                                u8 is_output)
3243 {
3244   snat_main_t *sm = &snat_main;
3245   u32 next_worker_index = sm->first_worker_index;
3246   u32 hash;
3247
3248   clib_bihash_kv_16_8_t kv16, value16;
3249   snat_main_per_thread_data_t *tsm;
3250   udp_header_t *udp;
3251
3252   if (PREDICT_FALSE (is_output))
3253     {
3254       u32 fib_index = sm->outside_fib_index;
3255       nat_outside_fib_t *outside_fib;
3256       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3257       fib_prefix_t pfx = {
3258         .fp_proto = FIB_PROTOCOL_IP4,
3259         .fp_len = 32,
3260         .fp_addr = {
3261                     .ip4.as_u32 = ip->dst_address.as_u32,
3262                     }
3263         ,
3264       };
3265
3266       udp = ip4_next_header (ip);
3267
3268       switch (vec_len (sm->outside_fibs))
3269         {
3270         case 0:
3271           fib_index = sm->outside_fib_index;
3272           break;
3273         case 1:
3274           fib_index = sm->outside_fibs[0].fib_index;
3275           break;
3276         default:
3277             /* *INDENT-OFF* */
3278             vec_foreach (outside_fib, sm->outside_fibs)
3279               {
3280                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3281                 if (FIB_NODE_INDEX_INVALID != fei)
3282                   {
3283                     if (fib_entry_get_resolving_interface (fei) != ~0)
3284                       {
3285                         fib_index = outside_fib->fib_index;
3286                         break;
3287                       }
3288                   }
3289               }
3290             /* *INDENT-ON* */
3291           break;
3292         }
3293
3294       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3295                  udp->dst_port, fib_index, ip->protocol);
3296
3297       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3298                                                   &kv16, &value16)))
3299         {
3300           tsm =
3301             vec_elt_at_index (sm->per_thread_data,
3302                               ed_value_get_thread_index (&value16));
3303           next_worker_index += tsm->thread_index;
3304
3305           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3306                                   next_worker_index, fib_index,
3307                                   clib_net_to_host_u32 (ip->
3308                                                         src_address.as_u32),
3309                                   clib_net_to_host_u32 (ip->
3310                                                         dst_address.as_u32));
3311
3312           return next_worker_index;
3313         }
3314     }
3315
3316   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3317     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3318
3319   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3320     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3321   else
3322     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3323
3324   if (PREDICT_TRUE (!is_output))
3325     {
3326       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3327                               next_worker_index, rx_fib_index,
3328                               clib_net_to_host_u32 (ip->src_address.as_u32),
3329                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3330     }
3331   else
3332     {
3333       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3334                               next_worker_index, rx_fib_index,
3335                               clib_net_to_host_u32 (ip->src_address.as_u32),
3336                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3337     }
3338
3339   return next_worker_index;
3340 }
3341
3342 static u32
3343 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3344                                u32 rx_fib_index, u8 is_output)
3345 {
3346   snat_main_t *sm = &snat_main;
3347   clib_bihash_kv_8_8_t kv, value;
3348   clib_bihash_kv_16_8_t kv16, value16;
3349   snat_main_per_thread_data_t *tsm;
3350
3351   u32 proto, next_worker_index = 0;
3352   udp_header_t *udp;
3353   u16 port;
3354   snat_static_mapping_t *m;
3355   u32 hash;
3356
3357   proto = ip_proto_to_nat_proto (ip->protocol);
3358
3359   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3360     {
3361       udp = ip4_next_header (ip);
3362
3363       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3364                  udp->src_port, rx_fib_index, ip->protocol);
3365
3366       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3367                                                   &kv16, &value16)))
3368         {
3369           tsm =
3370             vec_elt_at_index (sm->per_thread_data,
3371                               ed_value_get_thread_index (&value16));
3372           vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
3373             ed_value_get_session_index (&value16);
3374           next_worker_index = sm->first_worker_index + tsm->thread_index;
3375           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3376                                   next_worker_index, rx_fib_index,
3377                                   clib_net_to_host_u32 (ip->
3378                                                         src_address.as_u32),
3379                                   clib_net_to_host_u32 (ip->
3380                                                         dst_address.as_u32));
3381           return next_worker_index;
3382         }
3383     }
3384   else if (proto == NAT_PROTOCOL_ICMP)
3385     {
3386       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
3387         {
3388           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3389                                                       &kv16, &value16)))
3390             {
3391               tsm =
3392                 vec_elt_at_index (sm->per_thread_data,
3393                                   ed_value_get_thread_index (&value16));
3394               next_worker_index = sm->first_worker_index + tsm->thread_index;
3395               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3396                                       next_worker_index, rx_fib_index,
3397                                       clib_net_to_host_u32 (ip->
3398                                                             src_address.as_u32),
3399                                       clib_net_to_host_u32 (ip->
3400                                                             dst_address.as_u32));
3401               return next_worker_index;
3402             }
3403         }
3404     }
3405
3406   /* first try static mappings without port */
3407   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3408     {
3409       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3410       if (!clib_bihash_search_8_8
3411           (&sm->static_mapping_by_external, &kv, &value))
3412         {
3413           m = pool_elt_at_index (sm->static_mappings, value.value);
3414           next_worker_index = m->workers[0];
3415           goto done;
3416         }
3417     }
3418
3419   /* unknown protocol */
3420   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3421     {
3422       /* use current thread */
3423       next_worker_index = vlib_get_thread_index ();
3424       goto done;
3425     }
3426
3427   udp = ip4_next_header (ip);
3428   port = udp->dst_port;
3429
3430   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3431     {
3432       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3433       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3434       if (!icmp_type_is_error_message
3435           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3436         port = vnet_buffer (b)->ip.reass.l4_src_port;
3437       else
3438         {
3439           /* if error message, then it's not fragmented and we can access it */
3440           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3441           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3442           void *l4_header = ip4_next_header (inner_ip);
3443           switch (proto)
3444             {
3445             case NAT_PROTOCOL_ICMP:
3446               icmp = (icmp46_header_t *) l4_header;
3447               echo = (icmp_echo_header_t *) (icmp + 1);
3448               port = echo->identifier;
3449               break;
3450             case NAT_PROTOCOL_UDP:
3451             case NAT_PROTOCOL_TCP:
3452               port = ((tcp_udp_header_t *) l4_header)->src_port;
3453               break;
3454             default:
3455               next_worker_index = vlib_get_thread_index ();
3456               goto done;
3457             }
3458         }
3459     }
3460
3461   /* try static mappings with port */
3462   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3463     {
3464       init_nat_k (&kv, ip->dst_address, proto, 0, port);
3465       if (!clib_bihash_search_8_8
3466           (&sm->static_mapping_by_external, &kv, &value))
3467         {
3468           m = pool_elt_at_index (sm->static_mappings, value.value);
3469           if (!is_lb_static_mapping (m))
3470             {
3471               next_worker_index = m->workers[0];
3472               goto done;
3473             }
3474
3475           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3476             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3477
3478           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3479             next_worker_index =
3480               m->workers[hash & (_vec_len (m->workers) - 1)];
3481           else
3482             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3483           goto done;
3484         }
3485     }
3486
3487   /* worker by outside port */
3488   next_worker_index = sm->first_worker_index;
3489   next_worker_index +=
3490     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3491
3492 done:
3493   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3494                           clib_net_to_host_u32 (ip->src_address.as_u32),
3495                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3496   return next_worker_index;
3497 }
3498
3499 void
3500 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3501                 ip4_address_t * out_addr, u16 out_port,
3502                 ip4_address_t * eh_addr, u16 eh_port,
3503                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3504                 u32 fib_index, u16 flags, u32 thread_index)
3505 {
3506   snat_main_t *sm = &snat_main;
3507   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3508   snat_user_t *u;
3509   snat_session_t *s;
3510   clib_bihash_kv_8_8_t kv;
3511   vlib_main_t *vm = vlib_get_main ();
3512   f64 now = vlib_time_now (vm);
3513   nat_outside_fib_t *outside_fib;
3514   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3515   fib_prefix_t pfx = {
3516     .fp_proto = FIB_PROTOCOL_IP4,
3517     .fp_len = 32,
3518     .fp_addr = {
3519                 .ip4.as_u32 = eh_addr->as_u32,
3520                 },
3521   };
3522
3523   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3524     {
3525       if (nat_set_outside_address_and_port
3526           (sm->addresses, thread_index, *out_addr, out_port, proto))
3527         return;
3528     }
3529
3530   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3531   if (!u)
3532     return;
3533
3534   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3535   if (!s)
3536     return;
3537
3538   if (sm->endpoint_dependent)
3539     {
3540       nat_ed_lru_insert (tsm, s, now, nat_proto_to_ip_proto (proto));
3541     }
3542
3543   s->out2in.addr.as_u32 = out_addr->as_u32;
3544   s->out2in.port = out_port;
3545   s->nat_proto = proto;
3546   s->last_heard = now;
3547   s->flags = flags;
3548   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3549   s->ext_host_port = eh_port;
3550   user_session_increment (sm, u, snat_is_session_static (s));
3551   switch (vec_len (sm->outside_fibs))
3552     {
3553     case 0:
3554       s->out2in.fib_index = sm->outside_fib_index;
3555       break;
3556     case 1:
3557       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3558       break;
3559     default:
3560       /* *INDENT-OFF* */
3561       vec_foreach (outside_fib, sm->outside_fibs)
3562         {
3563           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3564           if (FIB_NODE_INDEX_INVALID != fei)
3565             {
3566               if (fib_entry_get_resolving_interface (fei) != ~0)
3567                 {
3568                   s->out2in.fib_index = outside_fib->fib_index;
3569                   break;
3570                 }
3571             }
3572         }
3573       /* *INDENT-ON* */
3574       break;
3575     }
3576   init_nat_o2i_kv (&kv, s, s - tsm->sessions);
3577   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3578     nat_elog_warn ("out2in key add failed");
3579
3580   s->in2out.addr.as_u32 = in_addr->as_u32;
3581   s->in2out.port = in_port;
3582   s->in2out.fib_index = fib_index;
3583   init_nat_i2o_kv (&kv, s, s - tsm->sessions);
3584   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3585     nat_elog_warn ("in2out key add failed");
3586 }
3587
3588 void
3589 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3590                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3591                 u32 ti)
3592 {
3593   snat_main_t *sm = &snat_main;
3594   clib_bihash_kv_8_8_t kv, value;
3595   u32 thread_index;
3596   snat_session_t *s;
3597   snat_main_per_thread_data_t *tsm;
3598
3599   if (sm->num_workers > 1)
3600     thread_index =
3601       sm->first_worker_index +
3602       (sm->workers[(clib_net_to_host_u16 (out_port) -
3603                     1024) / sm->port_per_thread]);
3604   else
3605     thread_index = sm->num_workers;
3606   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3607
3608   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3609   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3610     return;
3611
3612   s = pool_elt_at_index (tsm->sessions, value.value);
3613   nat_free_session_data (sm, s, thread_index, 1);
3614   nat44_delete_session (sm, s, thread_index);
3615 }
3616
3617 void
3618 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3619                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3620                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3621 {
3622   snat_main_t *sm = &snat_main;
3623   clib_bihash_kv_8_8_t kv, value;
3624   snat_session_t *s;
3625   snat_main_per_thread_data_t *tsm;
3626
3627   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3628
3629   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3630   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3631     return;
3632
3633   s = pool_elt_at_index (tsm->sessions, value.value);
3634   s->total_pkts = total_pkts;
3635   s->total_bytes = total_bytes;
3636 }
3637
3638 void
3639 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3640                    ip4_address_t * out_addr, u16 out_port,
3641                    ip4_address_t * eh_addr, u16 eh_port,
3642                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3643                    u32 fib_index, u16 flags, u32 thread_index)
3644 {
3645   snat_main_t *sm = &snat_main;
3646   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3647   snat_session_t *s;
3648   clib_bihash_kv_16_8_t kv;
3649   vlib_main_t *vm = vlib_get_main ();
3650   f64 now = vlib_time_now (vm);
3651   nat_outside_fib_t *outside_fib;
3652   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3653   fib_prefix_t pfx = {
3654     .fp_proto = FIB_PROTOCOL_IP4,
3655     .fp_len = 32,
3656     .fp_addr = {
3657                 .ip4.as_u32 = eh_addr->as_u32,
3658                 },
3659   };
3660
3661
3662   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3663     {
3664       if (nat_set_outside_address_and_port
3665           (sm->addresses, thread_index, *out_addr, out_port, proto))
3666         return;
3667     }
3668
3669   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3670     {
3671       if (nat_set_outside_address_and_port
3672           (sm->addresses, thread_index, *ehn_addr, ehn_port, proto))
3673         return;
3674     }
3675
3676   s = nat_ed_session_alloc (sm, thread_index, now, proto);
3677   if (!s)
3678     return;
3679
3680   s->last_heard = now;
3681   s->flags = flags;
3682   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3683   s->ext_host_nat_port = s->ext_host_port = eh_port;
3684   if (is_twice_nat_session (s))
3685     {
3686       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3687       s->ext_host_nat_port = ehn_port;
3688     }
3689   switch (vec_len (sm->outside_fibs))
3690     {
3691     case 0:
3692       s->out2in.fib_index = sm->outside_fib_index;
3693       break;
3694     case 1:
3695       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3696       break;
3697     default:
3698       /* *INDENT-OFF* */
3699       vec_foreach (outside_fib, sm->outside_fibs)
3700         {
3701           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3702           if (FIB_NODE_INDEX_INVALID != fei)
3703             {
3704               if (fib_entry_get_resolving_interface (fei) != ~0)
3705                 {
3706                   s->out2in.fib_index = outside_fib->fib_index;
3707                   break;
3708                 }
3709             }
3710         }
3711       /* *INDENT-ON* */
3712       break;
3713     }
3714   s->nat_proto = proto;
3715   s->out2in.addr.as_u32 = out_addr->as_u32;
3716   s->out2in.port = out_port;
3717
3718   s->in2out.addr.as_u32 = in_addr->as_u32;
3719   s->in2out.port = in_port;
3720   s->in2out.fib_index = fib_index;
3721
3722   init_ed_kv (&kv, *in_addr, in_port, s->ext_host_nat_addr,
3723               s->ext_host_nat_port, fib_index, nat_proto_to_ip_proto (proto),
3724               thread_index, s - tsm->sessions);
3725   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3726     nat_elog_warn ("in2out key add failed");
3727
3728   init_ed_kv (&kv, *out_addr, out_port, *eh_addr, eh_port,
3729               s->out2in.fib_index, nat_proto_to_ip_proto (proto),
3730               thread_index, s - tsm->sessions);
3731   if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 1))
3732     nat_elog_warn ("out2in key add failed");
3733 }
3734
3735 void
3736 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3737                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3738                    u32 fib_index, u32 ti)
3739 {
3740   snat_main_t *sm = &snat_main;
3741   clib_bihash_kv_16_8_t kv, value;
3742   u32 thread_index;
3743   snat_session_t *s;
3744   snat_main_per_thread_data_t *tsm;
3745
3746   if (sm->num_workers > 1)
3747     thread_index =
3748       sm->first_worker_index +
3749       (sm->workers[(clib_net_to_host_u16 (out_port) -
3750                     1024) / sm->port_per_thread]);
3751   else
3752     thread_index = sm->num_workers;
3753   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3754
3755   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3756   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3757     return;
3758
3759   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3760   nat_free_session_data (sm, s, thread_index, 1);
3761   nat44_delete_session (sm, s, thread_index);
3762 }
3763
3764 void
3765 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3766                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3767                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3768                    u32 thread_index)
3769 {
3770   snat_main_t *sm = &snat_main;
3771   clib_bihash_kv_16_8_t kv, value;
3772   snat_session_t *s;
3773   snat_main_per_thread_data_t *tsm;
3774
3775   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3776
3777   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3778   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3779     return;
3780
3781   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3782   s->total_pkts = total_pkts;
3783   s->total_bytes = total_bytes;
3784 }
3785
3786 static u32
3787 nat_calc_bihash_buckets (u32 n_elts)
3788 {
3789   return 1 << (max_log2 (n_elts >> 1) + 1);
3790 }
3791
3792 static u32
3793 nat_calc_bihash_memory (u32 n_buckets, uword kv_size)
3794 {
3795   return n_buckets * (8 + kv_size * 4);
3796 }
3797
3798 void
3799 nat44_db_init (snat_main_per_thread_data_t * tsm)
3800 {
3801   snat_main_t *sm = &snat_main;
3802
3803   pool_alloc (tsm->sessions, sm->max_translations_per_thread);
3804   pool_alloc (tsm->lru_pool, sm->max_translations_per_thread);
3805
3806   dlist_elt_t *head;
3807
3808   pool_get (tsm->lru_pool, head);
3809   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3810   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3811
3812   pool_get (tsm->lru_pool, head);
3813   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3814   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3815
3816   pool_get (tsm->lru_pool, head);
3817   tsm->udp_lru_head_index = head - tsm->lru_pool;
3818   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3819
3820   pool_get (tsm->lru_pool, head);
3821   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3822   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3823
3824   pool_get (tsm->lru_pool, head);
3825   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3826   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3827
3828   if (sm->endpoint_dependent)
3829     {
3830       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3831                              sm->translation_buckets,
3832                              sm->translation_memory_size);
3833       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3834                                           format_ed_session_kvp);
3835     }
3836   else
3837     {
3838       clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3839                             sm->translation_buckets,
3840                             sm->translation_memory_size);
3841       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
3842       clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3843                             sm->translation_buckets,
3844                             sm->translation_memory_size);
3845       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
3846     }
3847
3848   // TODO: resolve static mappings (put only to !ED)
3849   pool_alloc (tsm->list_pool, sm->max_translations_per_thread);
3850   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets,
3851                         sm->user_memory_size);
3852   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3853 }
3854
3855 void
3856 nat44_db_free (snat_main_per_thread_data_t * tsm)
3857 {
3858   snat_main_t *sm = &snat_main;
3859
3860   pool_free (tsm->sessions);
3861   pool_free (tsm->lru_pool);
3862
3863   if (sm->endpoint_dependent)
3864     {
3865       clib_bihash_free_16_8 (&tsm->in2out_ed);
3866     }
3867   else
3868     {
3869       clib_bihash_free_8_8 (&tsm->in2out);
3870       clib_bihash_free_8_8 (&tsm->out2in);
3871     }
3872
3873   // TODO: resolve static mappings (put only to !ED)
3874   pool_free (tsm->users);
3875   pool_free (tsm->list_pool);
3876   clib_bihash_free_8_8 (&tsm->user_hash);
3877 }
3878
3879 void
3880 nat44_sessions_clear ()
3881 {
3882   snat_main_t *sm = &snat_main;
3883   snat_main_per_thread_data_t *tsm;
3884
3885   if (sm->endpoint_dependent)
3886     {
3887       clib_bihash_free_16_8 (&sm->out2in_ed);
3888       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
3889                              clib_max (1, sm->num_workers) *
3890                              sm->translation_buckets,
3891                              clib_max (1, sm->num_workers) *
3892                              sm->translation_memory_size);
3893       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
3894                                           format_ed_session_kvp);
3895     }
3896
3897   /* *INDENT-OFF* */
3898   vec_foreach (tsm, sm->per_thread_data)
3899     {
3900       u32 ti;
3901
3902       nat44_db_free (tsm);
3903       nat44_db_init (tsm);
3904
3905       ti = tsm->snat_thread_index;
3906       vlib_set_simple_counter (&sm->total_users, ti, 0, 0);
3907       vlib_set_simple_counter (&sm->total_sessions, ti, 0, 0);
3908     }
3909   /* *INDENT-ON* */
3910 }
3911
3912 static clib_error_t *
3913 snat_config (vlib_main_t * vm, unformat_input_t * input)
3914 {
3915   snat_main_t *sm = &snat_main;
3916   nat66_main_t *nm = &nat66_main;
3917   snat_main_per_thread_data_t *tsm;
3918
3919   u32 static_mapping_buckets = 1024;
3920   uword static_mapping_memory_size = 64 << 20;
3921
3922   u32 nat64_bib_buckets = 1024;
3923   u32 nat64_bib_memory_size = 128 << 20;
3924
3925   u32 nat64_st_buckets = 2048;
3926   uword nat64_st_memory_size = 256 << 20;
3927
3928   u32 max_users_per_thread = 0;
3929   u32 user_memory_size = 0;
3930   u32 max_translations_per_thread = 0;
3931   u32 translation_memory_size = 0;
3932
3933   u32 max_translations_per_user = ~0;
3934
3935   u32 outside_vrf_id = 0;
3936   u32 outside_ip6_vrf_id = 0;
3937   u32 inside_vrf_id = 0;
3938   u8 static_mapping_only = 0;
3939   u8 static_mapping_connection_tracking = 0;
3940
3941   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3942   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3943   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3944   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3945
3946   sm->deterministic = 0;
3947   sm->out2in_dpo = 0;
3948   sm->endpoint_dependent = 0;
3949
3950   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3951     {
3952       if (unformat
3953           (input, "max translations per thread %d",
3954            &max_translations_per_thread))
3955         ;
3956       else if (unformat (input, "udp timeout %d", &udp_timeout))
3957         ;
3958       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3959         ;
3960       else if (unformat (input, "tcp transitory timeout %d",
3961                          &tcp_transitory_timeout));
3962       else if (unformat (input, "tcp established timeout %d",
3963                          &tcp_established_timeout));
3964       else if (unformat (input, "translation hash memory %d",
3965                          &translation_memory_size));
3966       else
3967         if (unformat
3968             (input, "max users per thread %d", &max_users_per_thread))
3969         ;
3970       else if (unformat (input, "user hash memory %d", &user_memory_size))
3971         ;
3972       else if (unformat (input, "max translations per user %d",
3973                          &max_translations_per_user))
3974         ;
3975       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3976         ;
3977       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3978         ;
3979       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3980         ;
3981       else if (unformat (input, "static mapping only"))
3982         {
3983           static_mapping_only = 1;
3984           if (unformat (input, "connection tracking"))
3985             static_mapping_connection_tracking = 1;
3986         }
3987       else if (unformat (input, "deterministic"))
3988         sm->deterministic = 1;
3989       else if (unformat (input, "nat64 bib hash buckets %d",
3990                          &nat64_bib_buckets))
3991         ;
3992       else if (unformat (input, "nat64 bib hash memory %d",
3993                          &nat64_bib_memory_size))
3994         ;
3995       else
3996         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3997         ;
3998       else if (unformat (input, "nat64 st hash memory %d",
3999                          &nat64_st_memory_size))
4000         ;
4001       else if (unformat (input, "out2in dpo"))
4002         sm->out2in_dpo = 1;
4003       else if (unformat (input, "endpoint-dependent"))
4004         sm->endpoint_dependent = 1;
4005       else
4006         return clib_error_return (0, "unknown input '%U'",
4007                                   format_unformat_error, input);
4008     }
4009
4010   if (sm->deterministic && sm->endpoint_dependent)
4011     return clib_error_return (0,
4012                               "deterministic and endpoint-dependent modes are mutually exclusive");
4013
4014   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
4015     return clib_error_return (0,
4016                               "static mapping only mode available only for simple nat");
4017
4018   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
4019     return clib_error_return (0,
4020                               "out2in dpo mode available only for simple nat");
4021   if (sm->endpoint_dependent && max_users_per_thread > 0)
4022     {
4023       return clib_error_return (0,
4024                                 "setting 'max users' in endpoint-dependent mode is not supported");
4025     }
4026
4027   if (sm->endpoint_dependent && max_translations_per_user != ~0)
4028     {
4029       return clib_error_return (0,
4030                                 "setting 'max translations per user' in endpoint-dependent mode is not supported");
4031     }
4032
4033   /* optionally configurable timeouts for testing purposes */
4034   sm->udp_timeout = udp_timeout;
4035   sm->tcp_transitory_timeout = tcp_transitory_timeout;
4036   sm->tcp_established_timeout = tcp_established_timeout;
4037   sm->icmp_timeout = icmp_timeout;
4038
4039   if (0 == max_users_per_thread)
4040     {
4041       max_users_per_thread = 1024;
4042     }
4043   sm->max_users_per_thread = max_users_per_thread;
4044   sm->user_buckets = nat_calc_bihash_buckets (sm->max_users_per_thread);
4045
4046   if (0 == max_translations_per_thread)
4047     {
4048       // default value based on legacy setting of load factor 10 * default
4049       // translation buckets 1024
4050       max_translations_per_thread = 10 * 1024;
4051     }
4052   sm->max_translations_per_thread = max_translations_per_thread;
4053   sm->translation_buckets =
4054     nat_calc_bihash_buckets (sm->max_translations_per_thread);
4055   if (0 == translation_memory_size)
4056     {
4057       translation_memory_size =
4058         nat_calc_bihash_memory (sm->translation_buckets,
4059                                 sizeof (clib_bihash_16_8_t));
4060     }
4061   sm->translation_memory_size = translation_memory_size;
4062   if (0 == user_memory_size)
4063     {
4064       user_memory_size =
4065         nat_calc_bihash_memory (sm->max_users_per_thread,
4066                                 sizeof (clib_bihash_8_8_t));
4067     }
4068   sm->user_memory_size = user_memory_size;
4069   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
4070
4071   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4072     sm->max_translations_per_thread : max_translations_per_user;
4073
4074   sm->outside_vrf_id = outside_vrf_id;
4075   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4076                                                              outside_vrf_id,
4077                                                              nat_fib_src_hi);
4078   nm->outside_vrf_id = outside_ip6_vrf_id;
4079   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
4080                                                              outside_ip6_vrf_id,
4081                                                              nat_fib_src_hi);
4082   sm->inside_vrf_id = inside_vrf_id;
4083   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4084                                                             inside_vrf_id,
4085                                                             nat_fib_src_hi);
4086   sm->static_mapping_only = static_mapping_only;
4087   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4088
4089   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4090                   nat64_st_memory_size);
4091
4092   if (sm->deterministic)
4093     {
4094       sm->in2out_node_index = snat_det_in2out_node.index;
4095       sm->in2out_output_node_index = ~0;
4096       sm->out2in_node_index = snat_det_out2in_node.index;
4097       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
4098       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
4099     }
4100   else
4101     {
4102       if (sm->endpoint_dependent)
4103         {
4104           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4105           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4106
4107           sm->in2out_node_index = nat44_ed_in2out_node.index;
4108           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4109           sm->out2in_node_index = nat44_ed_out2in_node.index;
4110
4111           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4112           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4113           nat_affinity_init (vm);
4114           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4115                        nat_ha_sref_ed_cb);
4116           clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4117                                  clib_max (1, sm->num_workers) *
4118                                  sm->translation_buckets,
4119                                  clib_max (1, sm->num_workers) *
4120                                  sm->translation_memory_size);
4121           clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4122                                               format_ed_session_kvp);
4123         }
4124       else
4125         {
4126           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4127           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4128
4129           sm->in2out_node_index = snat_in2out_node.index;
4130           sm->in2out_output_node_index = snat_in2out_output_node.index;
4131           sm->out2in_node_index = snat_out2in_node.index;
4132
4133           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4134           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4135           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4136         }
4137       if (!static_mapping_only ||
4138           (static_mapping_only && static_mapping_connection_tracking))
4139         {
4140           /* *INDENT-OFF* */
4141           vec_foreach (tsm, sm->per_thread_data)
4142             {
4143               nat44_db_init (tsm);
4144             }
4145           /* *INDENT-ON* */
4146         }
4147       else
4148         {
4149           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4150           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4151         }
4152       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4153                             "static_mapping_by_local", static_mapping_buckets,
4154                             static_mapping_memory_size);
4155       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4156                                          format_static_mapping_kvp);
4157
4158       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4159                             "static_mapping_by_external",
4160                             static_mapping_buckets,
4161                             static_mapping_memory_size);
4162       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4163                                          format_static_mapping_kvp);
4164     }
4165
4166   return 0;
4167 }
4168
4169 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4170
4171 static void
4172 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4173                                  uword opaque,
4174                                  u32 sw_if_index,
4175                                  ip4_address_t * address,
4176                                  u32 address_length,
4177                                  u32 if_address_index, u32 is_delete)
4178 {
4179   snat_main_t *sm = &snat_main;
4180   snat_static_map_resolve_t *rp;
4181   snat_static_mapping_t *m;
4182   clib_bihash_kv_8_8_t kv, value;
4183   int i, rv;
4184   ip4_address_t l_addr;
4185
4186   for (i = 0; i < vec_len (sm->to_resolve); i++)
4187     {
4188       rp = sm->to_resolve + i;
4189       if (rp->addr_only == 0)
4190         continue;
4191       if (rp->sw_if_index == sw_if_index)
4192         goto match;
4193     }
4194
4195   return;
4196
4197 match:
4198   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
4199               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
4200   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4201     m = 0;
4202   else
4203     m = pool_elt_at_index (sm->static_mappings, value.value);
4204
4205   if (!is_delete)
4206     {
4207       /* Don't trip over lease renewal, static config */
4208       if (m)
4209         return;
4210     }
4211   else
4212     {
4213       if (!m)
4214         return;
4215     }
4216
4217   /* Indetity mapping? */
4218   if (rp->l_addr.as_u32 == 0)
4219     l_addr.as_u32 = address[0].as_u32;
4220   else
4221     l_addr.as_u32 = rp->l_addr.as_u32;
4222   /* Add the static mapping */
4223   rv = snat_add_static_mapping (l_addr,
4224                                 address[0],
4225                                 rp->l_port,
4226                                 rp->e_port,
4227                                 rp->vrf_id,
4228                                 rp->addr_only, ~0 /* sw_if_index */ ,
4229                                 rp->proto, !is_delete, rp->twice_nat,
4230                                 rp->out2in_only, rp->tag, rp->identity_nat);
4231   if (rv)
4232     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4233 }
4234
4235 static void
4236 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4237                                        uword opaque,
4238                                        u32 sw_if_index,
4239                                        ip4_address_t * address,
4240                                        u32 address_length,
4241                                        u32 if_address_index, u32 is_delete)
4242 {
4243   snat_main_t *sm = &snat_main;
4244   snat_static_map_resolve_t *rp;
4245   ip4_address_t l_addr;
4246   int i, j;
4247   int rv;
4248   u8 twice_nat = 0;
4249   snat_address_t *addresses = sm->addresses;
4250
4251   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4252     {
4253       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4254         goto match;
4255     }
4256
4257   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4258     {
4259       twice_nat = 1;
4260       addresses = sm->twice_nat_addresses;
4261       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4262         goto match;
4263     }
4264
4265   return;
4266
4267 match:
4268   if (!is_delete)
4269     {
4270       /* Don't trip over lease renewal, static config */
4271       for (j = 0; j < vec_len (addresses); j++)
4272         if (addresses[j].addr.as_u32 == address->as_u32)
4273           return;
4274
4275       (void) snat_add_address (sm, address, ~0, twice_nat);
4276       /* Scan static map resolution vector */
4277       for (j = 0; j < vec_len (sm->to_resolve); j++)
4278         {
4279           rp = sm->to_resolve + j;
4280           if (rp->addr_only)
4281             continue;
4282           /* On this interface? */
4283           if (rp->sw_if_index == sw_if_index)
4284             {
4285               /* Indetity mapping? */
4286               if (rp->l_addr.as_u32 == 0)
4287                 l_addr.as_u32 = address[0].as_u32;
4288               else
4289                 l_addr.as_u32 = rp->l_addr.as_u32;
4290               /* Add the static mapping */
4291               rv = snat_add_static_mapping (l_addr,
4292                                             address[0],
4293                                             rp->l_port,
4294                                             rp->e_port,
4295                                             rp->vrf_id,
4296                                             rp->addr_only,
4297                                             ~0 /* sw_if_index */ ,
4298                                             rp->proto,
4299                                             rp->is_add, rp->twice_nat,
4300                                             rp->out2in_only, rp->tag,
4301                                             rp->identity_nat);
4302               if (rv)
4303                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4304                                     "i4", rv);
4305             }
4306         }
4307       return;
4308     }
4309   else
4310     {
4311       (void) snat_del_address (sm, address[0], 1, twice_nat);
4312       return;
4313     }
4314 }
4315
4316
4317 int
4318 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4319                             u8 twice_nat)
4320 {
4321   ip4_main_t *ip4_main = sm->ip4_main;
4322   ip4_address_t *first_int_addr;
4323   snat_static_map_resolve_t *rp;
4324   u32 *indices_to_delete = 0;
4325   int i, j;
4326   u32 *auto_add_sw_if_indices =
4327     twice_nat ? sm->
4328     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4329
4330   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4331     );
4332
4333   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4334     {
4335       if (auto_add_sw_if_indices[i] == sw_if_index)
4336         {
4337           if (is_del)
4338             {
4339               /* if have address remove it */
4340               if (first_int_addr)
4341                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4342               else
4343                 {
4344                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4345                     {
4346                       rp = sm->to_resolve + j;
4347                       if (rp->sw_if_index == sw_if_index)
4348                         vec_add1 (indices_to_delete, j);
4349                     }
4350                   if (vec_len (indices_to_delete))
4351                     {
4352                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4353                         vec_del1 (sm->to_resolve, j);
4354                       vec_free (indices_to_delete);
4355                     }
4356                 }
4357               if (twice_nat)
4358                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4359               else
4360                 vec_del1 (sm->auto_add_sw_if_indices, i);
4361             }
4362           else
4363             return VNET_API_ERROR_VALUE_EXIST;
4364
4365           return 0;
4366         }
4367     }
4368
4369   if (is_del)
4370     return VNET_API_ERROR_NO_SUCH_ENTRY;
4371
4372   /* add to the auto-address list */
4373   if (twice_nat)
4374     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4375   else
4376     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4377
4378   /* If the address is already bound - or static - add it now */
4379   if (first_int_addr)
4380     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4381
4382   return 0;
4383 }
4384
4385 int
4386 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4387                    nat_protocol_t proto, u32 vrf_id, int is_in)
4388 {
4389   snat_main_per_thread_data_t *tsm;
4390   clib_bihash_kv_8_8_t kv, value;
4391   ip4_header_t ip;
4392   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4393   snat_session_t *s;
4394   clib_bihash_8_8_t *t;
4395
4396   if (sm->endpoint_dependent)
4397     return VNET_API_ERROR_UNSUPPORTED;
4398
4399   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4400   if (sm->num_workers > 1)
4401     tsm =
4402       vec_elt_at_index (sm->per_thread_data,
4403                         sm->worker_in2out_cb (&ip, fib_index, 0));
4404   else
4405     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4406
4407   init_nat_k (&kv, *addr, port, fib_index, proto);
4408   t = is_in ? &tsm->in2out : &tsm->out2in;
4409   if (!clib_bihash_search_8_8 (t, &kv, &value))
4410     {
4411       if (pool_is_free_index (tsm->sessions, value.value))
4412         return VNET_API_ERROR_UNSPECIFIED;
4413
4414       s = pool_elt_at_index (tsm->sessions, value.value);
4415       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4416       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4417       return 0;
4418     }
4419
4420   return VNET_API_ERROR_NO_SUCH_ENTRY;
4421 }
4422
4423 int
4424 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4425                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4426                       u32 vrf_id, int is_in)
4427 {
4428   ip4_header_t ip;
4429   clib_bihash_16_8_t *t;
4430   clib_bihash_kv_16_8_t kv, value;
4431   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4432   snat_session_t *s;
4433   snat_main_per_thread_data_t *tsm;
4434
4435   if (!sm->endpoint_dependent)
4436     return VNET_API_ERROR_FEATURE_DISABLED;
4437
4438   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4439   if (sm->num_workers > 1)
4440     tsm =
4441       vec_elt_at_index (sm->per_thread_data,
4442                         sm->worker_in2out_cb (&ip, fib_index, 0));
4443   else
4444     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4445
4446   t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
4447   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
4448   if (clib_bihash_search_16_8 (t, &kv, &value))
4449     {
4450       return VNET_API_ERROR_NO_SUCH_ENTRY;
4451     }
4452
4453   if (pool_is_free_index (tsm->sessions, value.value))
4454     return VNET_API_ERROR_UNSPECIFIED;
4455   s = pool_elt_at_index (tsm->sessions, value.value);
4456   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4457   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
4458   return 0;
4459 }
4460
4461 void
4462 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4463 {
4464   snat_main_t *sm = &snat_main;
4465
4466   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4467   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4468   sm->psid = psid;
4469   sm->psid_offset = psid_offset;
4470   sm->psid_length = psid_length;
4471 }
4472
4473 void
4474 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4475 {
4476   snat_main_t *sm = &snat_main;
4477
4478   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4479   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4480   sm->start_port = start_port;
4481   sm->end_port = end_port;
4482 }
4483
4484 void
4485 nat_set_alloc_addr_and_port_default (void)
4486 {
4487   snat_main_t *sm = &snat_main;
4488
4489   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4490   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4491 }
4492
4493 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4494                                  vlib_node_runtime_t * node,
4495                                  vlib_frame_t * frame)
4496 {
4497   return 0;
4498 }
4499
4500 /* *INDENT-OFF* */
4501 VLIB_REGISTER_NODE (nat_default_node) = {
4502   .name = "nat-default",
4503   .vector_size = sizeof (u32),
4504   .format_trace = 0,
4505   .type = VLIB_NODE_TYPE_INTERNAL,
4506   .n_errors = 0,
4507   .n_next_nodes = NAT_N_NEXT,
4508   .next_nodes = {
4509     [NAT_NEXT_DROP] = "error-drop",
4510     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4511     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4512     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4513     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4514     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4515     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4516     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4517     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4518   },
4519 };
4520 /* *INDENT-ON* */
4521
4522 /*
4523  * fd.io coding-style-patch-verification: ON
4524  *
4525  * Local Variables:
4526  * eval: (c-set-style "gnu")
4527  * End:
4528  */