src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/lib/ipfix_logging.h>
  25 #include <nat/nat_inlines.h>
  26 #include <nat/nat44/inlines.h>
  27 #include <nat/nat_affinity.h>
  28 #include <nat/nat_syslog.h>
  29 #include <nat/nat_ha.h>
  30 #include <vnet/fib/fib_table.h>
  31 #include <vnet/fib/ip4_fib.h>
  32 #include <vnet/ip/reass/ip4_sv_reass.h>
  33 #include <vppinfra/bihash_16_8.h>
  34 #include <nat/nat44/ed_inlines.h>
  35 #include <vnet/ip/ip_table.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 /* *INDENT-OFF* */
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat-pre-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  47                                "ip4-sv-reassembly-feature"),
  48 };
  49 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  50   .arc_name = "ip4-unicast",
  51   .node_name = "nat-pre-out2in",
  52   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  53                                "ip4-dhcp-client-detect",
  54                                "ip4-sv-reassembly-feature"),
  55 };
  56 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  57   .arc_name = "ip4-unicast",
  58   .node_name = "nat44-in2out-worker-handoff",
  59   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  60 };
  61 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  62   .arc_name = "ip4-unicast",
  63   .node_name = "nat44-out2in-worker-handoff",
  64   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  65                                "ip4-dhcp-client-detect"),
  66 };
  67 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  68   .arc_name = "ip4-unicast",
  69   .node_name = "nat44-in2out",
  70   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  71 };
  72 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  73   .arc_name = "ip4-unicast",
  74   .node_name = "nat44-out2in",
  75   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  76                                "ip4-dhcp-client-detect"),
  77 };
  78 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  79   .arc_name = "ip4-unicast",
  80   .node_name = "nat44-classify",
  81   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  82 };
  83 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-ed-in2out",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-ed-out2in",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  92                                "ip4-dhcp-client-detect"),
  93 };
  94 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-ed-classify",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  98 };
  99 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 100   .arc_name = "ip4-unicast",
 101   .node_name = "nat44-handoff-classify",
 102   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out-fast",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 108 };
 109 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in-fast",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-hairpin-dst",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-hairpin-dst",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 124 };
 125
 126 /* Hook up output features */
 127 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 128   .arc_name = "ip4-output",
 129   .node_name = "nat44-in2out-output",
 130   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 133   .arc_name = "ip4-output",
 134   .node_name = "nat44-in2out-output-worker-handoff",
 135   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 136 };
 137 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 138   .arc_name = "ip4-output",
 139   .node_name = "nat44-hairpin-src",
 140   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 141 };
 142 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 143   .arc_name = "ip4-output",
 144   .node_name = "nat-pre-in2out-output",
 145   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 146   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 147 };
 148 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-ed-in2out-output",
 151   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 152   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 153 };
 154 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 155   .arc_name = "ip4-output",
 156   .node_name = "nat44-ed-hairpin-src",
 157   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 158   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 159 };
 160
 161 /* Hook up ip4-local features */
 162 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 163 {
 164   .arc_name = "ip4-local",
 165   .node_name = "nat44-hairpinning",
 166   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 167 };
 168 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 169 {
 170   .arc_name = "ip4-local",
 171   .node_name = "nat44-ed-hairpinning",
 172   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 173 };
 174
 175
 176 VLIB_PLUGIN_REGISTER () = {
 177     .version = VPP_BUILD_VER,
 178     .description = "Network Address Translation (NAT)",
 179 };
 180 /* *INDENT-ON* */
 181
 182 static u32
 183 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
 184                                u32 rx_fib_index, u8 is_output);
 185
 186 static u32
 187 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
 188                                u8 is_output);
 189
 190 static u32
 191 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
 192                            u32 rx_fib_index0, u8 is_output);
 193
 194 static u32
 195 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
 196                            u8 is_output);
 197
 198 static u32 nat_calc_bihash_buckets (u32 n_elts);
 199
 200 u8 *
 201 format_session_kvp (u8 * s, va_list * args)
 202 {
 203   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 204
 205   s = format (s, "%U session-index %llu", format_snat_key, v->key, v->value);
 206
 207   return s;
 208 }
 209
 210 u8 *
 211 format_static_mapping_kvp (u8 * s, va_list * args)
 212 {
 213   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 214
 215   s = format (s, "%U static-mapping-index %llu",
 216               format_snat_key, v->key, v->value);
 217
 218   return s;
 219 }
 220
 221 u8 *
 222 format_user_kvp (u8 * s, va_list * args)
 223 {
 224   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 225   snat_user_key_t k;
 226
 227   k.as_u64 = v->key;
 228
 229   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
 230               k.fib_index, v->value);
 231
 232   return s;
 233 }
 234
 235 u8 *
 236 format_ed_session_kvp (u8 * s, va_list * args)
 237 {
 238   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 239
 240   u8 proto;
 241   u16 r_port, l_port;
 242   ip4_address_t l_addr, r_addr;
 243   u32 fib_index;
 244
 245   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 246   s =
 247     format (s,
 248             "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
 249             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 250             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 251             format_ip_protocol, proto, fib_index,
 252             ed_value_get_session_index (v), ed_value_get_thread_index (v));
 253
 254   return s;
 255 }
 256
 257 void
 258 nat44_ei_free_session_data (snat_main_t * sm, snat_session_t * s,
 259                             u32 thread_index, u8 is_ha)
 260 {
 261   clib_bihash_kv_8_8_t kv;
 262
 263   snat_main_per_thread_data_t *tsm =
 264     vec_elt_at_index (sm->per_thread_data, thread_index);
 265
 266   init_nat_i2o_k (&kv, s);
 267   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 268     nat_elog_warn ("in2out key del failed");
 269
 270   init_nat_o2i_k (&kv, s);
 271   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 272     nat_elog_warn ("out2in key del failed");
 273
 274   if (!is_ha)
 275     {
 276       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 277                                &s->in2out.addr, s->in2out.port,
 278                                &s->out2in.addr, s->out2in.port, s->nat_proto);
 279
 280       nat_ipfix_logging_nat44_ses_delete (thread_index,
 281                                           s->in2out.addr.as_u32,
 282                                           s->out2in.addr.as_u32,
 283                                           s->nat_proto,
 284                                           s->in2out.port,
 285                                           s->out2in.port,
 286                                           s->in2out.fib_index);
 287
 288       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 289                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 290                    thread_index);
 291
 292     }
 293
 294   if (snat_is_session_static (s))
 295     return;
 296
 297   snat_free_outside_address_and_port (sm->addresses, thread_index,
 298                                       &s->out2in.addr, s->out2in.port,
 299                                       s->nat_proto);
 300 }
 301
 302 static_always_inline void
 303 nat44_ei_user_del_sessions (snat_user_t * u, u32 thread_index)
 304 {
 305   dlist_elt_t *elt;
 306   snat_session_t *s;
 307
 308   snat_main_t *sm = &snat_main;
 309   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 310
 311   // get head
 312   elt = pool_elt_at_index (tsm->list_pool,
 313                            u->sessions_per_user_list_head_index);
 314   // get first element
 315   elt = pool_elt_at_index (tsm->list_pool, elt->next);
 316
 317   while (elt->value != ~0)
 318     {
 319       s = pool_elt_at_index (tsm->sessions, elt->value);
 320       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 321
 322       nat44_ei_free_session_data (sm, s, thread_index, 0);
 323       nat44_delete_session (sm, s, thread_index);
 324     }
 325 }
 326
 327 int
 328 nat44_ei_user_del (ip4_address_t * addr, u32 fib_index)
 329 {
 330   int rv = 1;
 331
 332   snat_main_t *sm = &snat_main;
 333   snat_main_per_thread_data_t *tsm;
 334
 335   snat_user_key_t user_key;
 336   clib_bihash_kv_8_8_t kv, value;
 337
 338   if (sm->endpoint_dependent)
 339     return rv;
 340
 341   user_key.addr.as_u32 = addr->as_u32;
 342   user_key.fib_index = fib_index;
 343   kv.key = user_key.as_u64;
 344
 345   if (sm->num_workers > 1)
 346     {
 347       /* *INDENT-OFF* */
 348       vec_foreach (tsm, sm->per_thread_data)
 349         {
 350           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 351             {
 352               nat44_ei_user_del_sessions (
 353                   pool_elt_at_index (tsm->users, value.value),
 354                   tsm->thread_index);
 355               rv = 0;
 356               break;
 357             }
 358         }
 359       /* *INDENT-ON* */
 360     }
 361   else
 362     {
 363       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 364       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 365         {
 366           nat44_ei_user_del_sessions (pool_elt_at_index
 367                                       (tsm->users, value.value),
 368                                       tsm->thread_index);
 369           rv = 0;
 370         }
 371     }
 372   return rv;
 373 }
 374
 375 void
 376 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 377                        u8 is_ha)
 378 {
 379   clib_bihash_kv_8_8_t kv;
 380   u8 proto;
 381   u16 r_port, l_port;
 382   ip4_address_t *l_addr, *r_addr;
 383   u32 fib_index = 0;
 384   clib_bihash_kv_16_8_t ed_kv;
 385   snat_main_per_thread_data_t *tsm =
 386     vec_elt_at_index (sm->per_thread_data, thread_index);
 387
 388   if (is_ed_session (s))
 389     {
 390       per_vrf_sessions_unregister_session (s, thread_index);
 391     }
 392
 393   if (is_fwd_bypass_session (s))
 394     {
 395       if (snat_is_unk_proto_session (s))
 396         {
 397           init_ed_k (&ed_kv, s->in2out.addr, 0, s->ext_host_addr, 0, 0,
 398                      s->in2out.port);
 399         }
 400       else
 401         {
 402           l_port = s->in2out.port;
 403           r_port = s->ext_host_port;
 404           l_addr = &s->in2out.addr;
 405           r_addr = &s->ext_host_addr;
 406           proto = nat_proto_to_ip_proto (s->nat_proto);
 407           fib_index = s->in2out.fib_index;
 408           init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index,
 409                      proto);
 410         }
 411       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 412         nat_elog_warn ("in2out_ed key del failed");
 413       return;
 414     }
 415
 416   /* session lookup tables */
 417   if (is_ed_session (s))
 418     {
 419       if (is_affinity_sessions (s))
 420         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 421                              s->nat_proto, s->out2in.port);
 422       l_addr = &s->out2in.addr;
 423       r_addr = &s->ext_host_addr;
 424       fib_index = s->out2in.fib_index;
 425       if (snat_is_unk_proto_session (s))
 426         {
 427           proto = s->in2out.port;
 428           r_port = 0;
 429           l_port = 0;
 430         }
 431       else
 432         {
 433           proto = nat_proto_to_ip_proto (s->nat_proto);
 434           l_port = s->out2in.port;
 435           r_port = s->ext_host_port;
 436         }
 437       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 438       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 439         nat_elog_warn ("out2in_ed key del failed");
 440       l_addr = &s->in2out.addr;
 441       fib_index = s->in2out.fib_index;
 442       if (!snat_is_unk_proto_session (s))
 443         l_port = s->in2out.port;
 444       if (is_twice_nat_session (s))
 445         {
 446           r_addr = &s->ext_host_nat_addr;
 447           r_port = s->ext_host_nat_port;
 448         }
 449       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 450       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 451         nat_elog_warn ("in2out_ed key del failed");
 452
 453       if (!is_ha)
 454         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 455                                &s->in2out.addr, s->in2out.port,
 456                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 457                                &s->out2in.addr, s->out2in.port,
 458                                &s->ext_host_addr, s->ext_host_port,
 459                                s->nat_proto, is_twice_nat_session (s));
 460     }
 461   else
 462     {
 463       init_nat_i2o_k (&kv, s);
 464       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 465         nat_elog_warn ("in2out key del failed");
 466       init_nat_o2i_k (&kv, s);
 467       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 468         nat_elog_warn ("out2in key del failed");
 469
 470       if (!is_ha)
 471         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 472                                  &s->in2out.addr, s->in2out.port,
 473                                  &s->out2in.addr, s->out2in.port,
 474                                  s->nat_proto);
 475     }
 476
 477   if (snat_is_unk_proto_session (s))
 478     return;
 479
 480   if (!is_ha)
 481     {
 482       /* log NAT event */
 483       nat_ipfix_logging_nat44_ses_delete (thread_index,
 484                                           s->in2out.addr.as_u32,
 485                                           s->out2in.addr.as_u32,
 486                                           s->nat_proto,
 487                                           s->in2out.port,
 488                                           s->out2in.port,
 489                                           s->in2out.fib_index);
 490
 491       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 492                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 493                    thread_index);
 494     }
 495
 496   /* Twice NAT address and port for external host */
 497   if (is_twice_nat_session (s))
 498     {
 499       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 500                                           thread_index,
 501                                           &s->ext_host_nat_addr,
 502                                           s->ext_host_nat_port, s->nat_proto);
 503     }
 504
 505   if (snat_is_session_static (s))
 506     return;
 507
 508   snat_free_outside_address_and_port (sm->addresses, thread_index,
 509                                       &s->out2in.addr, s->out2in.port,
 510                                       s->nat_proto);
 511 }
 512
 513 snat_user_t *
 514 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 515                         u32 thread_index)
 516 {
 517   snat_user_t *u = 0;
 518   snat_user_key_t user_key;
 519   clib_bihash_kv_8_8_t kv, value;
 520   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 521   dlist_elt_t *per_user_list_head_elt;
 522
 523   user_key.addr.as_u32 = addr->as_u32;
 524   user_key.fib_index = fib_index;
 525   kv.key = user_key.as_u64;
 526
 527   /* Ever heard of the "user" = src ip4 address before? */
 528   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 529     {
 530       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 531         {
 532           vlib_increment_simple_counter (&sm->user_limit_reached,
 533                                          thread_index, 0, 1);
 534           nat_elog_warn ("maximum user limit reached");
 535           return NULL;
 536         }
 537       /* no, make a new one */
 538       pool_get (tsm->users, u);
 539       clib_memset (u, 0, sizeof (*u));
 540
 541       u->addr.as_u32 = addr->as_u32;
 542       u->fib_index = fib_index;
 543
 544       pool_get (tsm->list_pool, per_user_list_head_elt);
 545
 546       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 547         tsm->list_pool;
 548
 549       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 550
 551       kv.value = u - tsm->users;
 552
 553       /* add user */
 554       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 555         {
 556           nat_elog_warn ("user_hash key add failed");
 557           nat44_delete_user_with_no_session (sm, u, thread_index);
 558           return NULL;
 559         }
 560
 561       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 562                                pool_elts (tsm->users));
 563     }
 564   else
 565     {
 566       u = pool_elt_at_index (tsm->users, value.value);
 567     }
 568
 569   return u;
 570 }
 571
 572 // only NAT EI
 573 snat_session_t *
 574 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 575                               u32 thread_index, f64 now)
 576 {
 577   snat_session_t *s;
 578   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 579   u32 oldest_per_user_translation_list_index, session_index;
 580   dlist_elt_t *oldest_per_user_translation_list_elt;
 581   dlist_elt_t *per_user_translation_list_elt;
 582
 583   /* Over quota? Recycle the least recently used translation */
 584   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 585     {
 586       oldest_per_user_translation_list_index =
 587         clib_dlist_remove_head (tsm->list_pool,
 588                                 u->sessions_per_user_list_head_index);
 589
 590       ASSERT (oldest_per_user_translation_list_index != ~0);
 591
 592       /* Add it back to the end of the LRU list */
 593       clib_dlist_addtail (tsm->list_pool,
 594                           u->sessions_per_user_list_head_index,
 595                           oldest_per_user_translation_list_index);
 596       /* Get the list element */
 597       oldest_per_user_translation_list_elt =
 598         pool_elt_at_index (tsm->list_pool,
 599                            oldest_per_user_translation_list_index);
 600
 601       /* Get the session index from the list element */
 602       session_index = oldest_per_user_translation_list_elt->value;
 603
 604       /* Get the session */
 605       s = pool_elt_at_index (tsm->sessions, session_index);
 606
 607       // TODO: ONLY EI version should be called
 608       nat_free_session_data (sm, s, thread_index, 0);
 609       if (snat_is_session_static (s))
 610         u->nstaticsessions--;
 611       else
 612         u->nsessions--;
 613       s->flags = 0;
 614       s->total_bytes = 0;
 615       s->total_pkts = 0;
 616       s->state = 0;
 617       s->ext_host_addr.as_u32 = 0;
 618       s->ext_host_port = 0;
 619       s->ext_host_nat_addr.as_u32 = 0;
 620       s->ext_host_nat_port = 0;
 621     }
 622   else
 623     {
 624       pool_get (tsm->sessions, s);
 625       clib_memset (s, 0, sizeof (*s));
 626
 627       /* Create list elts */
 628       pool_get (tsm->list_pool, per_user_translation_list_elt);
 629       clib_dlist_init (tsm->list_pool,
 630                        per_user_translation_list_elt - tsm->list_pool);
 631
 632       per_user_translation_list_elt->value = s - tsm->sessions;
 633       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 634       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 635
 636       clib_dlist_addtail (tsm->list_pool,
 637                           s->per_user_list_head_index,
 638                           per_user_translation_list_elt - tsm->list_pool);
 639
 640       s->user_index = u - tsm->users;
 641       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 642                                pool_elts (tsm->sessions));
 643     }
 644
 645   s->ha_last_refreshed = now;
 646
 647   return s;
 648 }
 649
 650 void
 651 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 652                           int is_add)
 653 {
 654   snat_main_t *sm = &snat_main;
 655   fib_prefix_t prefix = {
 656     .fp_len = p_len,
 657     .fp_proto = FIB_PROTOCOL_IP4,
 658     .fp_addr = {
 659                 .ip4.as_u32 = addr->as_u32,
 660                 },
 661   };
 662   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 663
 664   if (is_add)
 665     fib_table_entry_update_one_path (fib_index,
 666                                      &prefix,
 667                                      sm->fib_src_low,
 668                                      (FIB_ENTRY_FLAG_CONNECTED |
 669                                       FIB_ENTRY_FLAG_LOCAL |
 670                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 671                                      DPO_PROTO_IP4,
 672                                      NULL,
 673                                      sw_if_index,
 674                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 675   else
 676     fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 677 }
 678
 679 int
 680 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 681                   u8 twice_nat)
 682 {
 683   snat_address_t *ap;
 684   snat_interface_t *i;
 685   vlib_thread_main_t *tm = vlib_get_thread_main ();
 686
 687   if (twice_nat && !sm->endpoint_dependent)
 688     {
 689       nat_log_err ("unsupported");
 690       return VNET_API_ERROR_UNSUPPORTED;
 691     }
 692
 693   /* Check if address already exists */
 694   /* *INDENT-OFF* */
 695   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 696     {
 697       if (ap->addr.as_u32 == addr->as_u32)
 698         {
 699           nat_log_err ("address exist");
 700           return VNET_API_ERROR_VALUE_EXIST;
 701         }
 702     }
 703   /* *INDENT-ON* */
 704
 705   if (twice_nat)
 706     vec_add2 (sm->twice_nat_addresses, ap, 1);
 707   else
 708     vec_add2 (sm->addresses, ap, 1);
 709
 710   ap->addr = *addr;
 711   if (vrf_id != ~0)
 712     ap->fib_index =
 713       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 714                                          sm->fib_src_low);
 715   else
 716     ap->fib_index = ~0;
 717
 718   /* *INDENT-OFF* */
 719   #define _(N, i, n, s) \
 720     clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 721     ap->busy_##n##_ports = 0; \
 722     ap->busy_##n##_ports_per_thread = 0;\
 723     vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 724     foreach_nat_protocol
 725   #undef _
 726   /* *INDENT-ON* */
 727
 728   if (twice_nat)
 729     return 0;
 730
 731   /* Add external address to FIB */
 732   /* *INDENT-OFF* */
 733   pool_foreach (i, sm->interfaces,
 734   ({
 735     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 736       continue;
 737
 738     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 739     break;
 740   }));
 741   pool_foreach (i, sm->output_feature_interfaces,
 742   ({
 743     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 744       continue;
 745
 746     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 747     break;
 748   }));
 749   /* *INDENT-ON* */
 750
 751   return 0;
 752 }
 753
 754 static int
 755 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 756 {
 757   snat_static_mapping_t *m;
 758   /* *INDENT-OFF* */
 759   pool_foreach (m, sm->static_mappings,
 760   ({
 761       if (is_addr_only_static_mapping (m) ||
 762           is_out2in_only_static_mapping (m) ||
 763           is_identity_static_mapping (m))
 764         continue;
 765       if (m->external_addr.as_u32 == addr.as_u32)
 766         return 1;
 767   }));
 768   /* *INDENT-ON* */
 769
 770   return 0;
 771 }
 772
 773 static void
 774 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 775                                        ip4_address_t l_addr,
 776                                        u16 l_port,
 777                                        u32 sw_if_index,
 778                                        u16 e_port,
 779                                        u32 vrf_id,
 780                                        nat_protocol_t proto,
 781                                        int addr_only, int is_add, u8 * tag,
 782                                        int twice_nat, int out2in_only,
 783                                        int identity_nat,
 784                                        ip4_address_t pool_addr, int exact)
 785 {
 786   snat_static_map_resolve_t *rp;
 787
 788   vec_add2 (sm->to_resolve, rp, 1);
 789   rp->l_addr.as_u32 = l_addr.as_u32;
 790   rp->l_port = l_port;
 791   rp->sw_if_index = sw_if_index;
 792   rp->e_port = e_port;
 793   rp->vrf_id = vrf_id;
 794   rp->proto = proto;
 795   rp->addr_only = addr_only;
 796   rp->is_add = is_add;
 797   rp->twice_nat = twice_nat;
 798   rp->out2in_only = out2in_only;
 799   rp->identity_nat = identity_nat;
 800   rp->tag = vec_dup (tag);
 801   rp->pool_addr = pool_addr;
 802   rp->exact = exact;
 803 }
 804
 805 static u32
 806 get_thread_idx_by_port (u16 e_port)
 807 {
 808   snat_main_t *sm = &snat_main;
 809   u32 thread_idx = sm->num_workers;
 810   if (sm->num_workers > 1)
 811     {
 812       thread_idx =
 813         sm->first_worker_index +
 814         sm->workers[(e_port - 1024) / sm->port_per_thread];
 815     }
 816   return thread_idx;
 817 }
 818
 819 void
 820 nat_ei_static_mapping_del_sessions (snat_main_t * sm,
 821                                     snat_main_per_thread_data_t * tsm,
 822                                     snat_user_key_t u_key, int addr_only,
 823                                     ip4_address_t e_addr, u16 e_port)
 824 {
 825   clib_bihash_kv_8_8_t kv, value;
 826   kv.key = u_key.as_u64;
 827   u64 user_index;
 828   dlist_elt_t *head, *elt;
 829   snat_user_t *u;
 830   snat_session_t *s;
 831   u32 elt_index, head_index, ses_index;
 832
 833   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 834     {
 835       user_index = value.value;
 836       u = pool_elt_at_index (tsm->users, user_index);
 837       if (u->nstaticsessions)
 838         {
 839           head_index = u->sessions_per_user_list_head_index;
 840           head = pool_elt_at_index (tsm->list_pool, head_index);
 841           elt_index = head->next;
 842           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 843           ses_index = elt->value;
 844           while (ses_index != ~0)
 845             {
 846               s = pool_elt_at_index (tsm->sessions, ses_index);
 847               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 848               ses_index = elt->value;
 849
 850               if (!addr_only)
 851                 {
 852                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 853                       (s->out2in.port != e_port))
 854                     continue;
 855                 }
 856
 857               if (is_lb_session (s))
 858                 continue;
 859
 860               if (!snat_is_session_static (s))
 861                 continue;
 862
 863               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 864               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 865
 866               if (!addr_only)
 867                 break;
 868             }
 869         }
 870     }
 871 }
 872
 873 void
 874 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 875                                     snat_main_per_thread_data_t * tsm,
 876                                     ip4_address_t l_addr,
 877                                     u16 l_port,
 878                                     u8 protocol,
 879                                     u32 fib_index, int addr_only,
 880                                     ip4_address_t e_addr, u16 e_port)
 881 {
 882   snat_session_t *s;
 883   u32 *indexes_to_free = NULL;
 884   /* *INDENT-OFF* */
 885   pool_foreach (s, tsm->sessions, {
 886     if (s->in2out.fib_index != fib_index ||
 887         s->in2out.addr.as_u32 != l_addr.as_u32)
 888       {
 889         continue;
 890       }
 891     if (!addr_only)
 892       {
 893         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 894             s->out2in.port != e_port ||
 895             s->in2out.port != l_port ||
 896             s->nat_proto != protocol)
 897           continue;
 898       }
 899
 900     if (is_lb_session (s))
 901       continue;
 902     if (!snat_is_session_static (s))
 903       continue;
 904     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 905     vec_add1 (indexes_to_free, s - tsm->sessions);
 906     if (!addr_only)
 907       break;
 908   });
 909   /* *INDENT-ON* */
 910   u32 *ses_index;
 911   vec_foreach (ses_index, indexes_to_free)
 912   {
 913     s = pool_elt_at_index (tsm->sessions, *ses_index);
 914     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 915   }
 916   vec_free (indexes_to_free);
 917 }
 918
 919 int
 920 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 921                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 922                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 923                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 924                          u8 identity_nat, ip4_address_t pool_addr, int exact)
 925 {
 926   snat_main_t *sm = &snat_main;
 927   snat_static_mapping_t *m;
 928   clib_bihash_kv_8_8_t kv, value;
 929   snat_address_t *a = 0;
 930   u32 fib_index = ~0;
 931   snat_interface_t *interface;
 932   int i;
 933   snat_main_per_thread_data_t *tsm;
 934   snat_user_key_t u_key;
 935   snat_user_t *u;
 936   dlist_elt_t *head, *elt;
 937   u32 elt_index, head_index;
 938   u32 ses_index;
 939   u64 user_index;
 940   snat_session_t *s;
 941   snat_static_map_resolve_t *rp, *rp_match = 0;
 942   nat44_lb_addr_port_t *local;
 943   u32 find = ~0;
 944
 945   if (!sm->endpoint_dependent)
 946     {
 947       if (twice_nat || out2in_only)
 948         return VNET_API_ERROR_UNSUPPORTED;
 949     }
 950
 951   /* If the external address is a specific interface address */
 952   if (sw_if_index != ~0)
 953     {
 954       ip4_address_t *first_int_addr;
 955
 956       for (i = 0; i < vec_len (sm->to_resolve); i++)
 957         {
 958           rp = sm->to_resolve + i;
 959           if (rp->sw_if_index != sw_if_index ||
 960               rp->l_addr.as_u32 != l_addr.as_u32 ||
 961               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 962             continue;
 963
 964           if (!addr_only)
 965             {
 966               if ((rp->l_port != l_port && rp->e_port != e_port)
 967                   || rp->proto != proto)
 968                 continue;
 969             }
 970
 971           rp_match = rp;
 972           break;
 973         }
 974
 975       /* Might be already set... */
 976       first_int_addr = ip4_interface_first_address
 977         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 978
 979       if (is_add)
 980         {
 981           if (rp_match)
 982             return VNET_API_ERROR_VALUE_EXIST;
 983
 984           snat_add_static_mapping_when_resolved
 985             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 986              addr_only, is_add, tag, twice_nat, out2in_only,
 987              identity_nat, pool_addr, exact);
 988
 989           /* DHCP resolution required? */
 990           if (first_int_addr == 0)
 991             {
 992               return 0;
 993             }
 994           else
 995             {
 996               e_addr.as_u32 = first_int_addr->as_u32;
 997               /* Identity mapping? */
 998               if (l_addr.as_u32 == 0)
 999                 l_addr.as_u32 = e_addr.as_u32;
1000             }
1001         }
1002       else
1003         {
1004           if (!rp_match)
1005             return VNET_API_ERROR_NO_SUCH_ENTRY;
1006
1007           vec_del1 (sm->to_resolve, i);
1008
1009           if (first_int_addr)
1010             {
1011               e_addr.as_u32 = first_int_addr->as_u32;
1012               /* Identity mapping? */
1013               if (l_addr.as_u32 == 0)
1014                 l_addr.as_u32 = e_addr.as_u32;
1015             }
1016           else
1017             return 0;
1018         }
1019     }
1020
1021   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
1022   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1023     m = 0;
1024   else
1025     m = pool_elt_at_index (sm->static_mappings, value.value);
1026
1027   if (is_add)
1028     {
1029       if (m)
1030         {
1031           if (is_identity_static_mapping (m))
1032             {
1033               /* *INDENT-OFF* */
1034               pool_foreach (local, m->locals,
1035               ({
1036                 if (local->vrf_id == vrf_id)
1037                   return VNET_API_ERROR_VALUE_EXIST;
1038               }));
1039               /* *INDENT-ON* */
1040               pool_get (m->locals, local);
1041               local->vrf_id = vrf_id;
1042               local->fib_index =
1043                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1044                                                    sm->fib_src_low);
1045               init_nat_kv (&kv, m->local_addr, m->local_port,
1046                            local->fib_index, m->proto,
1047                            m - sm->static_mappings);
1048               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1049               return 0;
1050             }
1051           else
1052             return VNET_API_ERROR_VALUE_EXIST;
1053         }
1054
1055       if (twice_nat && addr_only)
1056         return VNET_API_ERROR_UNSUPPORTED;
1057
1058       /* Convert VRF id to FIB index */
1059       if (vrf_id != ~0)
1060         fib_index =
1061           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1062                                              sm->fib_src_low);
1063       /* If not specified use inside VRF id from SNAT plugin startup config */
1064       else
1065         {
1066           fib_index = sm->inside_fib_index;
1067           vrf_id = sm->inside_vrf_id;
1068           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1069         }
1070
1071       if (!(out2in_only || identity_nat))
1072         {
1073           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
1074                       addr_only ? 0 : proto);
1075           if (!clib_bihash_search_8_8
1076               (&sm->static_mapping_by_local, &kv, &value))
1077             return VNET_API_ERROR_VALUE_EXIST;
1078         }
1079
1080       /* Find external address in allocated addresses and reserve port for
1081          address and port pair mapping when dynamic translations enabled */
1082       if (!(addr_only || sm->static_mapping_only || out2in_only))
1083         {
1084           for (i = 0; i < vec_len (sm->addresses); i++)
1085             {
1086               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1087                 {
1088                   a = sm->addresses + i;
1089                   /* External port must be unused */
1090                   switch (proto)
1091                     {
1092 #define _(N, j, n, s) \
1093                     case NAT_PROTOCOL_##N: \
1094                       if (a->busy_##n##_port_refcounts[e_port]) \
1095                         return VNET_API_ERROR_INVALID_VALUE; \
1096                       ++a->busy_##n##_port_refcounts[e_port]; \
1097                       if (e_port > 1024) \
1098                         { \
1099                           a->busy_##n##_ports++; \
1100                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1101                         } \
1102                       break;
1103                       foreach_nat_protocol
1104 #undef _
1105                     default:
1106                       nat_elog_info ("unknown protocol");
1107                       return VNET_API_ERROR_INVALID_VALUE_2;
1108                     }
1109                   break;
1110                 }
1111             }
1112           /* External address must be allocated */
1113           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1114             {
1115               if (sw_if_index != ~0)
1116                 {
1117                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1118                     {
1119                       rp = sm->to_resolve + i;
1120                       if (rp->addr_only)
1121                         continue;
1122                       if (rp->sw_if_index != sw_if_index &&
1123                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1124                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1125                           rp->e_port != e_port && rp->proto != proto)
1126                         continue;
1127
1128                       vec_del1 (sm->to_resolve, i);
1129                       break;
1130                     }
1131                 }
1132               return VNET_API_ERROR_NO_SUCH_ENTRY;
1133             }
1134         }
1135
1136       pool_get (sm->static_mappings, m);
1137       clib_memset (m, 0, sizeof (*m));
1138       m->tag = vec_dup (tag);
1139       m->local_addr = l_addr;
1140       m->external_addr = e_addr;
1141       m->twice_nat = twice_nat;
1142
1143       if (twice_nat == TWICE_NAT && exact)
1144         {
1145           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
1146           m->pool_addr = pool_addr;
1147         }
1148
1149       if (out2in_only)
1150         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1151       if (addr_only)
1152         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1153       if (identity_nat)
1154         {
1155           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1156           pool_get (m->locals, local);
1157           local->vrf_id = vrf_id;
1158           local->fib_index = fib_index;
1159         }
1160       else
1161         {
1162           m->vrf_id = vrf_id;
1163           m->fib_index = fib_index;
1164         }
1165       if (!addr_only)
1166         {
1167           m->local_port = l_port;
1168           m->external_port = e_port;
1169           m->proto = proto;
1170         }
1171
1172       if (sm->num_workers > 1)
1173         {
1174           ip4_header_t ip = {
1175             .src_address = m->local_addr,
1176           };
1177           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1178           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1179         }
1180       else
1181         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1182
1183       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
1184                    m - sm->static_mappings);
1185       if (!out2in_only)
1186         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1187
1188       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1189                    m - sm->static_mappings);
1190       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1191
1192       /* Delete dynamic sessions matching local address (+ local port) */
1193       // TODO: based on type of NAT EI/ED
1194       if (!(sm->static_mapping_only))
1195         {
1196           u_key.addr = m->local_addr;
1197           u_key.fib_index = m->fib_index;
1198           kv.key = u_key.as_u64;
1199           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1200             {
1201               user_index = value.value;
1202               u = pool_elt_at_index (tsm->users, user_index);
1203               if (u->nsessions)
1204                 {
1205                   head_index = u->sessions_per_user_list_head_index;
1206                   head = pool_elt_at_index (tsm->list_pool, head_index);
1207                   elt_index = head->next;
1208                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1209                   ses_index = elt->value;
1210                   while (ses_index != ~0)
1211                     {
1212                       s = pool_elt_at_index (tsm->sessions, ses_index);
1213                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1214                       ses_index = elt->value;
1215
1216                       if (snat_is_session_static (s))
1217                         continue;
1218
1219                       if (!addr_only && s->in2out.port != m->local_port)
1220                         continue;
1221
1222                       nat_free_session_data (sm, s,
1223                                              tsm - sm->per_thread_data, 0);
1224                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1225
1226                       if (!addr_only && !sm->endpoint_dependent)
1227                         break;
1228                     }
1229                 }
1230             }
1231         }
1232     }
1233   else
1234     {
1235       if (!m)
1236         {
1237           if (sw_if_index != ~0)
1238             return 0;
1239           else
1240             return VNET_API_ERROR_NO_SUCH_ENTRY;
1241         }
1242
1243       if (identity_nat)
1244         {
1245           if (vrf_id == ~0)
1246             vrf_id = sm->inside_vrf_id;
1247
1248           /* *INDENT-OFF* */
1249           pool_foreach (local, m->locals,
1250           ({
1251             if (local->vrf_id == vrf_id)
1252               find = local - m->locals;
1253           }));
1254           /* *INDENT-ON* */
1255           if (find == ~0)
1256             return VNET_API_ERROR_NO_SUCH_ENTRY;
1257
1258           local = pool_elt_at_index (m->locals, find);
1259           fib_index = local->fib_index;
1260           pool_put (m->locals, local);
1261         }
1262       else
1263         fib_index = m->fib_index;
1264
1265       /* Free external address port */
1266       if (!(addr_only || sm->static_mapping_only || out2in_only))
1267         {
1268           for (i = 0; i < vec_len (sm->addresses); i++)
1269             {
1270               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1271                 {
1272                   a = sm->addresses + i;
1273                   switch (proto)
1274                     {
1275 #define _(N, j, n, s) \
1276                     case NAT_PROTOCOL_##N: \
1277                       --a->busy_##n##_port_refcounts[e_port]; \
1278                       if (e_port > 1024) \
1279                         { \
1280                           a->busy_##n##_ports--; \
1281                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1282                         } \
1283                       break;
1284                       foreach_nat_protocol
1285 #undef _
1286                     default:
1287                       nat_elog_info ("unknown protocol");
1288                       return VNET_API_ERROR_INVALID_VALUE_2;
1289                     }
1290                   break;
1291                 }
1292             }
1293         }
1294
1295       if (sm->num_workers > 1)
1296         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1297       else
1298         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1299
1300       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1301       if (!out2in_only)
1302         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1303
1304       /* Delete session(s) for static mapping if exist */
1305       if (!(sm->static_mapping_only) ||
1306           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1307         {
1308           if (sm->endpoint_dependent)
1309             {
1310               nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1311                                                   m->local_port, m->proto,
1312                                                   fib_index, addr_only,
1313                                                   e_addr, e_port);
1314             }
1315           else
1316             {
1317               u_key.addr = m->local_addr;
1318               u_key.fib_index = fib_index;
1319               kv.key = u_key.as_u64;
1320               nat_ei_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1321                                                   e_addr, e_port);
1322             }
1323         }
1324
1325       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1326       if (pool_elts (m->locals))
1327         return 0;
1328
1329       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1330       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1331
1332       vec_free (m->tag);
1333       vec_free (m->workers);
1334       /* Delete static mapping from pool */
1335       pool_put (sm->static_mappings, m);
1336     }
1337
1338   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1339     return 0;
1340
1341   /* Add/delete external address to FIB */
1342   /* *INDENT-OFF* */
1343   pool_foreach (interface, sm->interfaces,
1344   ({
1345     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1346       continue;
1347
1348     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1349     break;
1350   }));
1351   pool_foreach (interface, sm->output_feature_interfaces,
1352   ({
1353     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1354       continue;
1355
1356     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1357     break;
1358   }));
1359   /* *INDENT-ON* */
1360
1361   return 0;
1362 }
1363
1364 int
1365 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1366                                  nat_protocol_t proto,
1367                                  nat44_lb_addr_port_t * locals, u8 is_add,
1368                                  twice_nat_type_t twice_nat, u8 out2in_only,
1369                                  u8 * tag, u32 affinity)
1370 {
1371   snat_main_t *sm = &snat_main;
1372   snat_static_mapping_t *m;
1373   clib_bihash_kv_8_8_t kv, value;
1374   snat_address_t *a = 0;
1375   int i;
1376   nat44_lb_addr_port_t *local;
1377   snat_main_per_thread_data_t *tsm;
1378   snat_session_t *s;
1379   uword *bitmap = 0;
1380
1381   if (!sm->endpoint_dependent)
1382     return VNET_API_ERROR_FEATURE_DISABLED;
1383
1384   init_nat_k (&kv, e_addr, e_port, 0, proto);
1385   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1386     m = 0;
1387   else
1388     m = pool_elt_at_index (sm->static_mappings, value.value);
1389
1390   if (is_add)
1391     {
1392       if (m)
1393         return VNET_API_ERROR_VALUE_EXIST;
1394
1395       if (vec_len (locals) < 2)
1396         return VNET_API_ERROR_INVALID_VALUE;
1397
1398       /* Find external address in allocated addresses and reserve port for
1399          address and port pair mapping when dynamic translations enabled */
1400       if (!(sm->static_mapping_only || out2in_only))
1401         {
1402           for (i = 0; i < vec_len (sm->addresses); i++)
1403             {
1404               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1405                 {
1406                   a = sm->addresses + i;
1407                   /* External port must be unused */
1408                   switch (proto)
1409                     {
1410 #define _(N, j, n, s) \
1411                     case NAT_PROTOCOL_##N: \
1412                       if (a->busy_##n##_port_refcounts[e_port]) \
1413                         return VNET_API_ERROR_INVALID_VALUE; \
1414                       ++a->busy_##n##_port_refcounts[e_port]; \
1415                       if (e_port > 1024) \
1416                         { \
1417                           a->busy_##n##_ports++; \
1418                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1419                         } \
1420                       break;
1421                       foreach_nat_protocol
1422 #undef _
1423                     default:
1424                       nat_elog_info ("unknown protocol");
1425                       return VNET_API_ERROR_INVALID_VALUE_2;
1426                     }
1427                   break;
1428                 }
1429             }
1430           /* External address must be allocated */
1431           if (!a)
1432             return VNET_API_ERROR_NO_SUCH_ENTRY;
1433         }
1434
1435       pool_get (sm->static_mappings, m);
1436       clib_memset (m, 0, sizeof (*m));
1437       m->tag = vec_dup (tag);
1438       m->external_addr = e_addr;
1439       m->external_port = e_port;
1440       m->proto = proto;
1441       m->twice_nat = twice_nat;
1442       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1443       if (out2in_only)
1444         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1445       m->affinity = affinity;
1446
1447       if (affinity)
1448         m->affinity_per_service_list_head_index =
1449           nat_affinity_get_per_service_list_head_index ();
1450       else
1451         m->affinity_per_service_list_head_index = ~0;
1452
1453       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1454                    m - sm->static_mappings);
1455       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1456         {
1457           nat_elog_err ("static_mapping_by_external key add failed");
1458           return VNET_API_ERROR_UNSPECIFIED;
1459         }
1460
1461       for (i = 0; i < vec_len (locals); i++)
1462         {
1463           locals[i].fib_index =
1464             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1465                                                locals[i].vrf_id,
1466                                                sm->fib_src_low);
1467           if (!out2in_only)
1468             {
1469               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1470                            locals[i].fib_index, m->proto,
1471                            m - sm->static_mappings);
1472               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1473             }
1474           locals[i].prefix = (i == 0) ? locals[i].probability :
1475             (locals[i - 1].prefix + locals[i].probability);
1476           pool_get (m->locals, local);
1477           *local = locals[i];
1478           if (sm->num_workers > 1)
1479             {
1480               ip4_header_t ip = {
1481                 .src_address = locals[i].addr,
1482               };
1483               bitmap =
1484                 clib_bitmap_set (bitmap,
1485                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1486                                  1);
1487             }
1488         }
1489
1490       /* Assign workers */
1491       if (sm->num_workers > 1)
1492         {
1493           /* *INDENT-OFF* */
1494           clib_bitmap_foreach (i, bitmap,
1495             ({
1496                vec_add1(m->workers, i);
1497             }));
1498           /* *INDENT-ON* */
1499         }
1500     }
1501   else
1502     {
1503       if (!m)
1504         return VNET_API_ERROR_NO_SUCH_ENTRY;
1505
1506       if (!is_lb_static_mapping (m))
1507         return VNET_API_ERROR_INVALID_VALUE;
1508
1509       /* Free external address port */
1510       if (!(sm->static_mapping_only || out2in_only))
1511         {
1512           for (i = 0; i < vec_len (sm->addresses); i++)
1513             {
1514               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1515                 {
1516                   a = sm->addresses + i;
1517                   switch (proto)
1518                     {
1519 #define _(N, j, n, s) \
1520                     case NAT_PROTOCOL_##N: \
1521                       --a->busy_##n##_port_refcounts[e_port]; \
1522                       if (e_port > 1024) \
1523                         { \
1524                           a->busy_##n##_ports--; \
1525                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1526                         } \
1527                       break;
1528                       foreach_nat_protocol
1529 #undef _
1530                     default:
1531                       nat_elog_info ("unknown protocol");
1532                       return VNET_API_ERROR_INVALID_VALUE_2;
1533                     }
1534                   break;
1535                 }
1536             }
1537         }
1538
1539       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1540       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1541         {
1542           nat_elog_err ("static_mapping_by_external key del failed");
1543           return VNET_API_ERROR_UNSPECIFIED;
1544         }
1545
1546       /* *INDENT-OFF* */
1547       pool_foreach (local, m->locals,
1548       ({
1549           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1550                             sm->fib_src_low);
1551           if (!out2in_only)
1552             {
1553 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1554               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1555                 {
1556                   nat_elog_err ("static_mapping_by_local key del failed");
1557                   return VNET_API_ERROR_UNSPECIFIED;
1558                 }
1559             }
1560
1561           if (sm->num_workers > 1)
1562             {
1563               ip4_header_t ip = {
1564                 .src_address = local->addr,
1565               };
1566               tsm = vec_elt_at_index (sm->per_thread_data,
1567                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1568             }
1569           else
1570             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1571
1572           /* Delete sessions */
1573           pool_foreach (s, tsm->sessions, {
1574             if (!(is_lb_session (s)))
1575               continue;
1576
1577             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1578                 s->in2out.port != local->port)
1579               continue;
1580
1581             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1582             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1583           });
1584       }));
1585       /* *INDENT-ON* */
1586       if (m->affinity)
1587         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1588       pool_free (m->locals);
1589       vec_free (m->tag);
1590       vec_free (m->workers);
1591
1592       pool_put (sm->static_mappings, m);
1593     }
1594
1595   return 0;
1596 }
1597
1598 int
1599 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1600                                        ip4_address_t l_addr, u16 l_port,
1601                                        nat_protocol_t proto, u32 vrf_id,
1602                                        u8 probability, u8 is_add)
1603 {
1604   snat_main_t *sm = &snat_main;
1605   snat_static_mapping_t *m = 0;
1606   clib_bihash_kv_8_8_t kv, value;
1607   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1608   snat_main_per_thread_data_t *tsm;
1609   snat_session_t *s;
1610   u32 *locals = 0;
1611   uword *bitmap = 0;
1612   int i;
1613
1614   if (!sm->endpoint_dependent)
1615     return VNET_API_ERROR_FEATURE_DISABLED;
1616
1617   init_nat_k (&kv, e_addr, e_port, 0, proto);
1618   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1619     m = pool_elt_at_index (sm->static_mappings, value.value);
1620
1621   if (!m)
1622     return VNET_API_ERROR_NO_SUCH_ENTRY;
1623
1624   if (!is_lb_static_mapping (m))
1625     return VNET_API_ERROR_INVALID_VALUE;
1626
1627   /* *INDENT-OFF* */
1628   pool_foreach (local, m->locals,
1629   ({
1630     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1631         (local->vrf_id == vrf_id))
1632       {
1633         match_local = local;
1634         break;
1635       }
1636   }));
1637   /* *INDENT-ON* */
1638
1639   if (is_add)
1640     {
1641       if (match_local)
1642         return VNET_API_ERROR_VALUE_EXIST;
1643
1644       pool_get (m->locals, local);
1645       clib_memset (local, 0, sizeof (*local));
1646       local->addr.as_u32 = l_addr.as_u32;
1647       local->port = l_port;
1648       local->probability = probability;
1649       local->vrf_id = vrf_id;
1650       local->fib_index =
1651         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1652                                            sm->fib_src_low);
1653
1654       if (!is_out2in_only_static_mapping (m))
1655         {
1656           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto,
1657                        m - sm->static_mappings);
1658           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1659             nat_elog_err ("static_mapping_by_local key add failed");
1660         }
1661     }
1662   else
1663     {
1664       if (!match_local)
1665         return VNET_API_ERROR_NO_SUCH_ENTRY;
1666
1667       if (pool_elts (m->locals) < 3)
1668         return VNET_API_ERROR_UNSPECIFIED;
1669
1670       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1671                         sm->fib_src_low);
1672
1673       if (!is_out2in_only_static_mapping (m))
1674         {
1675           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1676           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1677             nat_elog_err ("static_mapping_by_local key del failed");
1678         }
1679
1680       if (sm->num_workers > 1)
1681         {
1682           ip4_header_t ip = {
1683             .src_address = local->addr,
1684           };
1685           tsm = vec_elt_at_index (sm->per_thread_data,
1686                                   sm->worker_in2out_cb (&ip, m->fib_index,
1687                                                         0));
1688         }
1689       else
1690         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1691
1692       /* Delete sessions */
1693       /* *INDENT-OFF* */
1694       pool_foreach (s, tsm->sessions, {
1695         if (!(is_lb_session (s)))
1696           continue;
1697
1698         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1699             s->in2out.port != match_local->port)
1700           continue;
1701
1702         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1703         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1704       });
1705       /* *INDENT-ON* */
1706
1707       pool_put (m->locals, match_local);
1708     }
1709
1710   vec_free (m->workers);
1711
1712   /* *INDENT-OFF* */
1713   pool_foreach (local, m->locals,
1714   ({
1715     vec_add1 (locals, local - m->locals);
1716     if (sm->num_workers > 1)
1717       {
1718         ip4_header_t ip;
1719         ip.src_address.as_u32 = local->addr.as_u32,
1720         bitmap = clib_bitmap_set (bitmap,
1721                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1722                                   1);
1723       }
1724   }));
1725   /* *INDENT-ON* */
1726
1727   ASSERT (vec_len (locals) > 1);
1728
1729   local = pool_elt_at_index (m->locals, locals[0]);
1730   local->prefix = local->probability;
1731   for (i = 1; i < vec_len (locals); i++)
1732     {
1733       local = pool_elt_at_index (m->locals, locals[i]);
1734       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1735       local->prefix = local->probability + prev_local->prefix;
1736     }
1737
1738   /* Assign workers */
1739   if (sm->num_workers > 1)
1740     {
1741       /* *INDENT-OFF* */
1742       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1743       /* *INDENT-ON* */
1744     }
1745
1746   return 0;
1747 }
1748
1749 int
1750 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1751                   u8 twice_nat)
1752 {
1753   snat_address_t *a = 0;
1754   snat_session_t *ses;
1755   u32 *ses_to_be_removed = 0, *ses_index;
1756   snat_main_per_thread_data_t *tsm;
1757   snat_static_mapping_t *m;
1758   snat_interface_t *interface;
1759   int i;
1760   snat_address_t *addresses =
1761     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1762
1763   /* Find SNAT address */
1764   for (i = 0; i < vec_len (addresses); i++)
1765     {
1766       if (addresses[i].addr.as_u32 == addr.as_u32)
1767         {
1768           a = addresses + i;
1769           break;
1770         }
1771     }
1772   if (!a)
1773     {
1774       nat_log_err ("no such address");
1775       return VNET_API_ERROR_NO_SUCH_ENTRY;
1776     }
1777
1778   if (delete_sm)
1779     {
1780       ip4_address_t pool_addr = { 0 };
1781       /* *INDENT-OFF* */
1782       pool_foreach (m, sm->static_mappings,
1783       ({
1784           if (m->external_addr.as_u32 == addr.as_u32)
1785             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1786                                             m->local_port, m->external_port,
1787                                             m->vrf_id,
1788                                             is_addr_only_static_mapping(m), ~0,
1789                                             m->proto, 0 /* is_add */,
1790                                             m->twice_nat,
1791                                             is_out2in_only_static_mapping(m),
1792                                             m->tag,
1793                                             is_identity_static_mapping(m),
1794                                             pool_addr, 0);
1795       }));
1796       /* *INDENT-ON* */
1797     }
1798   else
1799     {
1800       /* Check if address is used in some static mapping */
1801       if (is_snat_address_used_in_static_mapping (sm, addr))
1802         {
1803           nat_log_err ("address used in static mapping");
1804           return VNET_API_ERROR_UNSPECIFIED;
1805         }
1806     }
1807
1808   if (a->fib_index != ~0)
1809     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1810
1811   /* Delete sessions using address */
1812   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1813     {
1814       vec_foreach (tsm, sm->per_thread_data)
1815       {
1816         /* *INDENT-OFF* */
1817         pool_foreach (ses, tsm->sessions, ({
1818           if (ses->out2in.addr.as_u32 == addr.as_u32)
1819             {
1820               nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1821               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1822             }
1823         }));
1824         /* *INDENT-ON* */
1825
1826         if (sm->endpoint_dependent)
1827           {
1828             vec_foreach (ses_index, ses_to_be_removed)
1829             {
1830               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1831               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1832             }
1833           }
1834         else
1835           {
1836             vec_foreach (ses_index, ses_to_be_removed)
1837             {
1838               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1839               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1840             }
1841           }
1842
1843         vec_free (ses_to_be_removed);
1844       }
1845     }
1846
1847 #define _(N, i, n, s) \
1848   vec_free (a->busy_##n##_ports_per_thread);
1849   foreach_nat_protocol
1850 #undef _
1851     if (twice_nat)
1852     {
1853       vec_del1 (sm->twice_nat_addresses, i);
1854       return 0;
1855     }
1856   else
1857     vec_del1 (sm->addresses, i);
1858
1859   /* Delete external address from FIB */
1860   /* *INDENT-OFF* */
1861   pool_foreach (interface, sm->interfaces,
1862   ({
1863     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1864       continue;
1865
1866     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1867     break;
1868   }));
1869   pool_foreach (interface, sm->output_feature_interfaces,
1870   ({
1871     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1872       continue;
1873
1874     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1875     break;
1876   }));
1877   /* *INDENT-ON* */
1878
1879   return 0;
1880 }
1881
1882 static void
1883 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1884 {
1885 #define _(x)                                                                  \
1886   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1887                                 sw_if_index);                                 \
1888   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1889   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1890                                 sw_if_index);                                 \
1891   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1892   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1893                                 sw_if_index);                                 \
1894   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1895   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1896                                 sw_if_index);                                 \
1897   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1898   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1899                                 sw_if_index);                                 \
1900   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1901   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1902                                 sw_if_index);                                 \
1903   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1904   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1905                                 sw_if_index);                                 \
1906   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1907   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1908                                 sw_if_index);                                 \
1909   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1910   foreach_nat_counter;
1911 #undef _
1912   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1913   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1914 }
1915
1916 void
1917 expire_per_vrf_sessions (u32 fib_index)
1918 {
1919   per_vrf_sessions_t *per_vrf_sessions;
1920   snat_main_per_thread_data_t *tsm;
1921   snat_main_t *sm = &snat_main;
1922
1923   /* *INDENT-OFF* */
1924   vec_foreach (tsm, sm->per_thread_data)
1925     {
1926       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1927         {
1928           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1929               (per_vrf_sessions->tx_fib_index == fib_index))
1930             {
1931               per_vrf_sessions->expired = 1;
1932             }
1933         }
1934     }
1935   /* *INDENT-ON* */
1936 }
1937
1938 void
1939 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1940 {
1941   snat_main_t *sm = &snat_main;
1942   nat_fib_t *fib;
1943
1944   // we don't care if it is outside/inside fib
1945   // we just care about their ref_count
1946   // if it reaches 0 sessions should expire
1947   // because the fib isn't valid for NAT anymore
1948
1949   vec_foreach (fib, sm->fibs)
1950   {
1951     if (fib->fib_index == fib_index)
1952       {
1953         if (is_del)
1954           {
1955             fib->ref_count--;
1956             if (!fib->ref_count)
1957               {
1958                 vec_del1 (sm->fibs, fib - sm->fibs);
1959                 expire_per_vrf_sessions (fib_index);
1960               }
1961             return;
1962           }
1963         else
1964           fib->ref_count++;
1965       }
1966   }
1967   if (!is_del)
1968     {
1969       vec_add2 (sm->fibs, fib, 1);
1970       fib->ref_count = 1;
1971       fib->fib_index = fib_index;
1972     }
1973 }
1974
1975 int
1976 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1977 {
1978   snat_main_t *sm = &snat_main;
1979   snat_interface_t *i;
1980   const char *feature_name, *del_feature_name;
1981   snat_address_t *ap;
1982   snat_static_mapping_t *m;
1983   nat_outside_fib_t *outside_fib;
1984   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1985                                                        sw_if_index);
1986
1987   if (!sm->enabled)
1988     {
1989       nat_log_err ("nat44 is disabled");
1990       return VNET_API_ERROR_UNSUPPORTED;
1991     }
1992
1993   if (sm->out2in_dpo && !is_inside)
1994     {
1995       nat_log_err ("error unsupported");
1996       return VNET_API_ERROR_UNSUPPORTED;
1997     }
1998
1999   /* *INDENT-OFF* */
2000   pool_foreach (i, sm->output_feature_interfaces,
2001   ({
2002     if (i->sw_if_index == sw_if_index)
2003       {
2004         nat_log_err ("error interface already configured");
2005         return VNET_API_ERROR_VALUE_EXIST;
2006       }
2007   }));
2008   /* *INDENT-ON* */
2009
2010   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2011     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
2012   else
2013     {
2014       if (sm->num_workers > 1)
2015         feature_name =
2016           is_inside ? "nat44-in2out-worker-handoff" :
2017           "nat44-out2in-worker-handoff";
2018       else if (sm->endpoint_dependent)
2019         {
2020           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
2021         }
2022       else
2023         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
2024     }
2025
2026   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
2027     sm->fq_in2out_index =
2028       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
2029
2030   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2031     sm->fq_out2in_index =
2032       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
2033
2034   if (sm->endpoint_dependent)
2035     update_per_vrf_sessions_vec (fib_index, is_del);
2036
2037   if (!is_inside)
2038     {
2039       /* *INDENT-OFF* */
2040       vec_foreach (outside_fib, sm->outside_fibs)
2041         {
2042           if (outside_fib->fib_index == fib_index)
2043             {
2044               if (is_del)
2045                 {
2046                   outside_fib->refcount--;
2047                   if (!outside_fib->refcount)
2048                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2049                 }
2050               else
2051                 outside_fib->refcount++;
2052               goto feature_set;
2053             }
2054         }
2055       /* *INDENT-ON* */
2056       if (!is_del)
2057         {
2058           vec_add2 (sm->outside_fibs, outside_fib, 1);
2059           outside_fib->refcount = 1;
2060           outside_fib->fib_index = fib_index;
2061         }
2062     }
2063
2064 feature_set:
2065   /* *INDENT-OFF* */
2066   pool_foreach (i, sm->interfaces,
2067   ({
2068     if (i->sw_if_index == sw_if_index)
2069       {
2070         if (is_del)
2071           {
2072             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
2073               {
2074                 if (is_inside)
2075                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
2076                 else
2077                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
2078
2079                 if (sm->num_workers > 1)
2080                   {
2081                     del_feature_name = "nat44-handoff-classify";
2082                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2083                                                  "nat44-out2in-worker-handoff";
2084                   }
2085                 else if (sm->endpoint_dependent)
2086                   {
2087                     del_feature_name = "nat44-ed-classify";
2088                     feature_name = !is_inside ?  "nat-pre-in2out" :
2089                                                  "nat-pre-out2in";
2090                   }
2091                 else
2092                   {
2093                     del_feature_name = "nat44-classify";
2094                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2095                   }
2096
2097                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2098                 if (rv)
2099                   return rv;
2100                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2101                                              sw_if_index, 0, 0, 0);
2102                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2103                                              sw_if_index, 1, 0, 0);
2104                 if (!is_inside)
2105                   {
2106                     if (sm->endpoint_dependent)
2107                       vnet_feature_enable_disable ("ip4-local",
2108                                                    "nat44-ed-hairpinning",
2109                                                    sw_if_index, 1, 0, 0);
2110                     else
2111                       vnet_feature_enable_disable ("ip4-local",
2112                                                    "nat44-hairpinning",
2113                                                    sw_if_index, 1, 0, 0);
2114                   }
2115               }
2116             else
2117               {
2118                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2119                 if (rv)
2120                   return rv;
2121                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2122                                              sw_if_index, 0, 0, 0);
2123                 pool_put (sm->interfaces, i);
2124                 if (is_inside)
2125                   {
2126                     if (sm->endpoint_dependent)
2127                       vnet_feature_enable_disable ("ip4-local",
2128                                                    "nat44-ed-hairpinning",
2129                                                    sw_if_index, 0, 0, 0);
2130                     else
2131                       vnet_feature_enable_disable ("ip4-local",
2132                                                    "nat44-hairpinning",
2133                                                    sw_if_index, 0, 0, 0);
2134                   }
2135               }
2136           }
2137         else
2138           {
2139             if ((nat_interface_is_inside(i) && is_inside) ||
2140                 (nat_interface_is_outside(i) && !is_inside))
2141               return 0;
2142
2143             if (sm->num_workers > 1)
2144               {
2145                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2146                                                  "nat44-out2in-worker-handoff";
2147                 feature_name = "nat44-handoff-classify";
2148               }
2149             else if (sm->endpoint_dependent)
2150               {
2151                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2152                                                  "nat-pre-out2in";
2153
2154                 feature_name = "nat44-ed-classify";
2155               }
2156             else
2157               {
2158                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2159                 feature_name = "nat44-classify";
2160               }
2161
2162             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2163             if (rv)
2164               return rv;
2165             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2166                                          sw_if_index, 0, 0, 0);
2167             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2168                                          sw_if_index, 1, 0, 0);
2169             if (!is_inside)
2170               {
2171                 if (sm->endpoint_dependent)
2172                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2173                                                sw_if_index, 0, 0, 0);
2174                 else
2175                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2176                                                sw_if_index, 0, 0, 0);
2177               }
2178             goto set_flags;
2179           }
2180
2181         goto fib;
2182       }
2183   }));
2184   /* *INDENT-ON* */
2185
2186   if (is_del)
2187     {
2188       nat_log_err ("error interface couldn't be found");
2189       return VNET_API_ERROR_NO_SUCH_ENTRY;
2190     }
2191
2192   pool_get (sm->interfaces, i);
2193   i->sw_if_index = sw_if_index;
2194   i->flags = 0;
2195   nat_validate_counters (sm, sw_if_index);
2196
2197   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2198                                0);
2199
2200   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2201   if (rv)
2202     return rv;
2203
2204   if (is_inside && !sm->out2in_dpo)
2205     {
2206       if (sm->endpoint_dependent)
2207         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2208                                      sw_if_index, 1, 0, 0);
2209       else
2210         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2211                                      sw_if_index, 1, 0, 0);
2212     }
2213
2214 set_flags:
2215   if (is_inside)
2216     {
2217       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2218       return 0;
2219     }
2220   else
2221     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2222
2223   /* Add/delete external addresses to FIB */
2224 fib:
2225   /* *INDENT-OFF* */
2226   vec_foreach (ap, sm->addresses)
2227     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2228
2229   pool_foreach (m, sm->static_mappings,
2230   ({
2231     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2232       continue;
2233
2234     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2235   }));
2236   /* *INDENT-ON* */
2237
2238   return 0;
2239 }
2240
2241 int
2242 snat_interface_add_del_output_feature (u32 sw_if_index,
2243                                        u8 is_inside, int is_del)
2244 {
2245   snat_main_t *sm = &snat_main;
2246   snat_interface_t *i;
2247   snat_address_t *ap;
2248   snat_static_mapping_t *m;
2249   nat_outside_fib_t *outside_fib;
2250   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2251                                                        sw_if_index);
2252
2253   if (!sm->enabled)
2254     {
2255       nat_log_err ("nat44 is disabled");
2256       return VNET_API_ERROR_UNSUPPORTED;
2257     }
2258
2259   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2260     {
2261       nat_log_err ("error unsupported");
2262       return VNET_API_ERROR_UNSUPPORTED;
2263     }
2264
2265   /* *INDENT-OFF* */
2266   pool_foreach (i, sm->interfaces,
2267   ({
2268     if (i->sw_if_index == sw_if_index)
2269       {
2270         nat_log_err ("error interface already configured");
2271         return VNET_API_ERROR_VALUE_EXIST;
2272       }
2273   }));
2274   /* *INDENT-ON* */
2275
2276   if (sm->endpoint_dependent)
2277     update_per_vrf_sessions_vec (fib_index, is_del);
2278
2279   if (!is_inside)
2280     {
2281       /* *INDENT-OFF* */
2282       vec_foreach (outside_fib, sm->outside_fibs)
2283         {
2284           if (outside_fib->fib_index == fib_index)
2285             {
2286               if (is_del)
2287                 {
2288                   outside_fib->refcount--;
2289                   if (!outside_fib->refcount)
2290                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2291                 }
2292               else
2293                 outside_fib->refcount++;
2294               goto feature_set;
2295             }
2296         }
2297       /* *INDENT-ON* */
2298       if (!is_del)
2299         {
2300           vec_add2 (sm->outside_fibs, outside_fib, 1);
2301           outside_fib->refcount = 1;
2302           outside_fib->fib_index = fib_index;
2303         }
2304     }
2305
2306 feature_set:
2307   if (is_inside)
2308     {
2309       if (sm->endpoint_dependent)
2310         {
2311           int rv =
2312             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2313           if (rv)
2314             return rv;
2315           rv =
2316             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2317                                                             !is_del);
2318           if (rv)
2319             return rv;
2320           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2321                                        sw_if_index, !is_del, 0, 0);
2322           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2323                                        sw_if_index, !is_del, 0, 0);
2324         }
2325       else
2326         {
2327           int rv =
2328             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2329           if (rv)
2330             return rv;
2331           rv =
2332             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2333                                                             !is_del);
2334           if (rv)
2335             return rv;
2336           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2337                                        sw_if_index, !is_del, 0, 0);
2338           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2339                                        sw_if_index, !is_del, 0, 0);
2340         }
2341       goto fq;
2342     }
2343
2344   if (sm->num_workers > 1)
2345     {
2346       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2347       if (rv)
2348         return rv;
2349       rv =
2350         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2351       if (rv)
2352         return rv;
2353       vnet_feature_enable_disable ("ip4-unicast",
2354                                    "nat44-out2in-worker-handoff",
2355                                    sw_if_index, !is_del, 0, 0);
2356       vnet_feature_enable_disable ("ip4-output",
2357                                    "nat44-in2out-output-worker-handoff",
2358                                    sw_if_index, !is_del, 0, 0);
2359     }
2360   else
2361     {
2362       if (sm->endpoint_dependent)
2363         {
2364           int rv =
2365             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2366           if (rv)
2367             return rv;
2368           rv =
2369             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2370                                                             !is_del);
2371           if (rv)
2372             return rv;
2373           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2374                                        sw_if_index, !is_del, 0, 0);
2375           vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2376                                        sw_if_index, !is_del, 0, 0);
2377         }
2378       else
2379         {
2380           int rv =
2381             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2382           if (rv)
2383             return rv;
2384           rv =
2385             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2386                                                             !is_del);
2387           if (rv)
2388             return rv;
2389           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2390                                        sw_if_index, !is_del, 0, 0);
2391           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2392                                        sw_if_index, !is_del, 0, 0);
2393         }
2394     }
2395
2396 fq:
2397   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2398     sm->fq_in2out_output_index =
2399       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2400
2401   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2402     sm->fq_out2in_index =
2403       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2404
2405   /* *INDENT-OFF* */
2406   pool_foreach (i, sm->output_feature_interfaces,
2407   ({
2408     if (i->sw_if_index == sw_if_index)
2409       {
2410         if (is_del)
2411           pool_put (sm->output_feature_interfaces, i);
2412         else
2413           return VNET_API_ERROR_VALUE_EXIST;
2414
2415         goto fib;
2416       }
2417   }));
2418   /* *INDENT-ON* */
2419
2420   if (is_del)
2421     {
2422       nat_log_err ("error interface couldn't be found");
2423       return VNET_API_ERROR_NO_SUCH_ENTRY;
2424     }
2425
2426   pool_get (sm->output_feature_interfaces, i);
2427   i->sw_if_index = sw_if_index;
2428   i->flags = 0;
2429   nat_validate_counters (sm, sw_if_index);
2430   if (is_inside)
2431     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2432   else
2433     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2434
2435   /* Add/delete external addresses to FIB */
2436 fib:
2437   if (is_inside)
2438     return 0;
2439
2440   /* *INDENT-OFF* */
2441   vec_foreach (ap, sm->addresses)
2442     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2443
2444   pool_foreach (m, sm->static_mappings,
2445   ({
2446     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2447       continue;
2448
2449     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2450   }));
2451   /* *INDENT-ON* */
2452
2453   return 0;
2454 }
2455
2456 int
2457 snat_set_workers (uword * bitmap)
2458 {
2459   snat_main_t *sm = &snat_main;
2460   int i, j = 0;
2461
2462   if (sm->num_workers < 2)
2463     return VNET_API_ERROR_FEATURE_DISABLED;
2464
2465   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2466     return VNET_API_ERROR_INVALID_WORKER;
2467
2468   vec_free (sm->workers);
2469   /* *INDENT-OFF* */
2470   clib_bitmap_foreach (i, bitmap,
2471     ({
2472       vec_add1(sm->workers, i);
2473       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2474       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2475       j++;
2476     }));
2477   /* *INDENT-ON* */
2478
2479   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2480
2481   return 0;
2482 }
2483
2484 static void
2485 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2486                          u32 sw_if_index, u32 new_fib_index,
2487                          u32 old_fib_index)
2488 {
2489   snat_main_t *sm = &snat_main;
2490   nat_outside_fib_t *outside_fib;
2491   snat_interface_t *i;
2492   u8 is_add = 1;
2493   u8 match = 0;
2494
2495   if (!sm->enabled || (new_fib_index == old_fib_index)
2496       || (!vec_len (sm->outside_fibs)))
2497     {
2498       return;
2499     }
2500
2501   /* *INDENT-OFF* */
2502   pool_foreach (i, sm->interfaces,
2503     ({
2504       if (i->sw_if_index == sw_if_index)
2505         {
2506           if (!(nat_interface_is_outside (i)))
2507             return;
2508           match = 1;
2509         }
2510     }));
2511
2512   pool_foreach (i, sm->output_feature_interfaces,
2513     ({
2514       if (i->sw_if_index == sw_if_index)
2515         {
2516           if (!(nat_interface_is_outside (i)))
2517             return;
2518           match = 1;
2519         }
2520     }));
2521   /* *INDENT-ON* */
2522
2523   if (!match)
2524     return;
2525
2526   vec_foreach (outside_fib, sm->outside_fibs)
2527   {
2528     if (outside_fib->fib_index == old_fib_index)
2529       {
2530         outside_fib->refcount--;
2531         if (!outside_fib->refcount)
2532           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2533         break;
2534       }
2535   }
2536
2537   vec_foreach (outside_fib, sm->outside_fibs)
2538   {
2539     if (outside_fib->fib_index == new_fib_index)
2540       {
2541         outside_fib->refcount++;
2542         is_add = 0;
2543         break;
2544       }
2545   }
2546
2547   if (is_add)
2548     {
2549       vec_add2 (sm->outside_fibs, outside_fib, 1);
2550       outside_fib->refcount = 1;
2551       outside_fib->fib_index = new_fib_index;
2552     }
2553 }
2554
2555 static void
2556 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2557                          u32 sw_if_index, u32 new_fib_index,
2558                          u32 old_fib_index);
2559
2560 static void
2561 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2562                                        uword opaque,
2563                                        u32 sw_if_index,
2564                                        ip4_address_t * address,
2565                                        u32 address_length,
2566                                        u32 if_address_index, u32 is_delete);
2567
2568 static void
2569 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2570                                  uword opaque,
2571                                  u32 sw_if_index,
2572                                  ip4_address_t * address,
2573                                  u32 address_length,
2574                                  u32 if_address_index, u32 is_delete);
2575
2576 static int
2577 nat_alloc_addr_and_port_default (snat_address_t * addresses, u32 fib_index,
2578                                  u32 thread_index, nat_protocol_t proto,
2579                                  ip4_address_t * addr, u16 * port,
2580                                  u16 port_per_thread, u32 snat_thread_index);
2581
2582 void
2583 test_key_calc_split ()
2584 {
2585   ip4_address_t l_addr;
2586   l_addr.as_u8[0] = 1;
2587   l_addr.as_u8[1] = 1;
2588   l_addr.as_u8[2] = 1;
2589   l_addr.as_u8[3] = 1;
2590   ip4_address_t r_addr;
2591   r_addr.as_u8[0] = 2;
2592   r_addr.as_u8[1] = 2;
2593   r_addr.as_u8[2] = 2;
2594   r_addr.as_u8[3] = 2;
2595   u16 l_port = 40001;
2596   u16 r_port = 40301;
2597   u8 proto = 9;
2598   u32 fib_index = 9000001;
2599   u32 thread_index = 3000000001;
2600   u32 session_index = 3000000221;
2601   clib_bihash_kv_16_8_t kv;
2602   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2603               thread_index, session_index);
2604   ip4_address_t l_addr2;
2605   ip4_address_t r_addr2;
2606   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2607   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2608   u16 l_port2 = 0;
2609   u16 r_port2 = 0;
2610   u8 proto2 = 0;
2611   u32 fib_index2 = 0;
2612   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2613                &r_port2);
2614   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2615   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2616   ASSERT (l_port == l_port2);
2617   ASSERT (r_port == r_port2);
2618   ASSERT (proto == proto2);
2619   ASSERT (fib_index == fib_index2);
2620   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2621   ASSERT (session_index == ed_value_get_session_index (&kv));
2622
2623   fib_index = 7001;
2624   proto = 5;
2625   nat_protocol_t proto3 = ~0;
2626   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2627   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2628   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2629   ASSERT (l_port == l_port2);
2630   ASSERT (proto == proto3);
2631   ASSERT (fib_index == fib_index2);
2632 }
2633
2634 static clib_error_t *
2635 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2636 {
2637   snat_main_t *sm = &snat_main;
2638   u32 fib_index;
2639
2640   if (sm->endpoint_dependent)
2641     {
2642       // TODO: consider removing all NAT interfaces
2643       if (!is_add)
2644         {
2645           fib_index = ip4_fib_index_from_table_id (table_id);
2646           if (fib_index != ~0)
2647             expire_per_vrf_sessions (fib_index);
2648         }
2649     }
2650   return 0;
2651 }
2652
2653 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2654
2655 void
2656 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2657 {
2658   vlib_node_t *node;
2659
2660   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2661   sm->ei_out2in_node_index = node->index;
2662   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2663   sm->ei_in2out_node_index = node->index;
2664   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2665   sm->ei_in2out_output_node_index = node->index;
2666
2667   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2668   sm->ed_out2in_node_index = node->index;
2669   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2670   sm->ed_in2out_node_index = node->index;
2671   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2672   sm->ed_in2out_output_node_index = node->index;
2673
2674   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2675   sm->error_node_index = node->index;
2676   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2677   sm->pre_in2out_node_index = node->index;
2678   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2679   sm->pre_out2in_node_index = node->index;
2680   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2681   sm->pre_in2out_node_index = node->index;
2682   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2683   sm->pre_out2in_node_index = node->index;
2684   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2685   sm->in2out_fast_node_index = node->index;
2686   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2687   sm->in2out_slowpath_node_index = node->index;
2688   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2689   sm->in2out_slowpath_output_node_index = node->index;
2690   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2691   sm->ed_in2out_slowpath_node_index = node->index;
2692   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2693   sm->out2in_fast_node_index = node->index;
2694   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2695   sm->ed_out2in_slowpath_node_index = node->index;
2696   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2697   sm->hairpinning_node_index = node->index;
2698   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2699   sm->hairpin_dst_node_index = node->index;
2700   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2701   sm->hairpin_src_node_index = node->index;
2702   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2703   sm->ed_hairpinning_node_index = node->index;
2704   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2705   sm->ed_hairpin_dst_node_index = node->index;
2706   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2707   sm->ed_hairpin_src_node_index = node->index;
2708 }
2709
2710 #define nat_init_simple_counter(c, n, sn) \
2711 do                                        \
2712   {                                       \
2713     c.name = n;                           \
2714     c.stat_segment_name = sn;             \
2715     vlib_validate_simple_counter (&c, 0); \
2716     vlib_zero_simple_counter (&c, 0);     \
2717   } while (0);
2718
2719 static clib_error_t *
2720 nat_init (vlib_main_t * vm)
2721 {
2722   snat_main_t *sm = &snat_main;
2723   vlib_thread_main_t *tm = vlib_get_thread_main ();
2724   vlib_thread_registration_t *tr;
2725   ip4_add_del_interface_address_callback_t cbi = { 0 };
2726   ip4_table_bind_callback_t cbt = { 0 };
2727   u32 i, num_threads = 0;
2728   uword *p, *bitmap = 0;
2729
2730   clib_memset (sm, 0, sizeof (*sm));
2731
2732   // required
2733   sm->vnet_main = vnet_get_main ();
2734   // convenience
2735   sm->ip4_main = &ip4_main;
2736   sm->api_main = vlibapi_get_main ();
2737   sm->ip4_lookup_main = &ip4_main.lookup_main;
2738
2739   // frame queue indices used for handoff
2740   sm->fq_out2in_index = ~0;
2741   sm->fq_in2out_index = ~0;
2742   sm->fq_in2out_output_index = ~0;
2743
2744   sm->log_level = SNAT_LOG_ERROR;
2745
2746   nat44_set_node_indexes (sm, vm);
2747   sm->log_class = vlib_log_register_class ("nat", 0);
2748   nat_ipfix_logging_init (vm);
2749
2750   nat_init_simple_counter (sm->total_users, "total-users",
2751                            "/nat44/total-users");
2752   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2753                            "/nat44/total-sessions");
2754   nat_init_simple_counter (sm->user_limit_reached, "user-limit-reached",
2755                            "/nat44/user-limit-reached");
2756
2757 #define _(x)                                            \
2758   sm->counters.fastpath.in2out.x.name = #x;             \
2759   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2760       "/nat44/in2out/fastpath/" #x;                     \
2761   sm->counters.slowpath.in2out.x.name = #x;             \
2762   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2763       "/nat44/in2out/slowpath/" #x;                     \
2764   sm->counters.fastpath.out2in.x.name = #x;             \
2765   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2766       "/nat44/out2in/fastpath/" #x;                     \
2767   sm->counters.slowpath.out2in.x.name = #x;             \
2768   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2769       "/nat44/out2in/slowpath/" #x;                     \
2770   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2771   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2772       "/nat44/ed/in2out/fastpath/" #x;                  \
2773   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2774   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2775       "/nat44/ed/in2out/slowpath/" #x;                  \
2776   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2777   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2778       "/nat44/ed/out2in/fastpath/" #x;                  \
2779   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2780   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2781       "/nat44/ed/out2in/slowpath/" #x;
2782   foreach_nat_counter;
2783 #undef _
2784   sm->counters.hairpinning.name = "hairpinning";
2785   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2786
2787   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2788   if (p)
2789     {
2790       tr = (vlib_thread_registration_t *) p[0];
2791       if (tr)
2792         {
2793           sm->num_workers = tr->count;
2794           sm->first_worker_index = tr->first_index;
2795         }
2796     }
2797   num_threads = tm->n_vlib_mains - 1;
2798   sm->port_per_thread = 0xffff - 1024;
2799   vec_validate (sm->per_thread_data, num_threads);
2800
2801   /* Use all available workers by default */
2802   if (sm->num_workers > 1)
2803     {
2804
2805       for (i = 0; i < sm->num_workers; i++)
2806         bitmap = clib_bitmap_set (bitmap, i, 1);
2807       snat_set_workers (bitmap);
2808       clib_bitmap_free (bitmap);
2809     }
2810   else
2811     sm->per_thread_data[0].snat_thread_index = 0;
2812
2813   /* callbacks to call when interface address changes. */
2814   cbi.function = snat_ip4_add_del_interface_address_cb;
2815   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2816   cbi.function = nat_ip4_add_del_addr_only_sm_cb;
2817   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2818
2819   /* callbacks to call when interface to table biding changes */
2820   cbt.function = snat_update_outside_fib;
2821   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2822
2823   sm->fib_src_low =
2824     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2825                          FIB_SOURCE_BH_SIMPLE);
2826   sm->fib_src_hi =
2827     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2828                          FIB_SOURCE_BH_SIMPLE);
2829
2830   /* used only by out2in-dpo feature */
2831   nat_dpo_module_init ();
2832
2833   nat_affinity_init (vm);
2834   nat_ha_init (vm, sm->num_workers, num_threads);
2835
2836   test_key_calc_split ();
2837   return nat44_api_hookup (vm);
2838 }
2839
2840 VLIB_INIT_FUNCTION (nat_init);
2841
2842 int
2843 nat44_plugin_enable (nat44_config_t c)
2844 {
2845   snat_main_t *sm = &snat_main;
2846   u32 static_mapping_buckets = 1024;
2847   u32 static_mapping_memory_size = 64 << 20;
2848
2849   if (sm->enabled)
2850     {
2851       nat_log_err ("nat44 is enabled");
2852       return 1;
2853     }
2854
2855   // c.static_mapping_only + c.connection_tracking
2856   //  - supported in NAT EI & NAT ED
2857   // c.out2in_dpo, c.static_mapping_only
2858   //  - supported in NAT EI
2859
2860   if (c.endpoint_dependent)
2861     {
2862       if ((c.static_mapping_only && !c.connection_tracking) || c.out2in_dpo)
2863         {
2864           nat_log_err ("unsupported combination of configuration");
2865           return 1;
2866         }
2867       if (c.users || c.user_sessions)
2868         {
2869           nat_log_err ("unsupported combination of configuration");
2870           return 1;
2871         }
2872     }
2873
2874   // reset to defaults:
2875   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2876   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2877   //
2878   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2879   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2880   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2881   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2882
2883   // nat44 feature configuration
2884   sm->endpoint_dependent = c.endpoint_dependent;
2885   sm->static_mapping_only = c.static_mapping_only;
2886   sm->static_mapping_connection_tracking = c.connection_tracking;
2887   sm->forwarding_enabled = 0;
2888   sm->mss_clamping = 0;
2889
2890   if (!c.users)
2891     c.users = 1024;
2892
2893   sm->max_users_per_thread = c.users;
2894   sm->user_buckets = nat_calc_bihash_buckets (c.users);
2895
2896   if (!c.sessions)
2897     c.sessions = 10 * 1024;
2898
2899   sm->max_translations_per_thread = c.sessions;
2900   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2901
2902   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2903   sm->max_translations_per_user
2904     = c.user_sessions ? c.user_sessions : sm->max_translations_per_thread;
2905
2906   sm->outside_vrf_id = c.outside_vrf;
2907   sm->outside_fib_index =
2908     fib_table_find_or_create_and_lock
2909     (FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2910
2911   sm->inside_vrf_id = c.inside_vrf;
2912   sm->inside_fib_index =
2913     fib_table_find_or_create_and_lock
2914     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2915
2916   if (c.endpoint_dependent)
2917     {
2918       sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2919       sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
2920       sm->out2in_node_index = sm->ed_out2in_node_index;
2921       sm->in2out_node_index = sm->ed_in2out_node_index;
2922       sm->in2out_output_node_index = sm->ed_in2out_output_node_index;
2923       sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
2924       sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
2925
2926       // try to move it into nat44_db_init,
2927       // consider static mapping requirements
2928       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
2929                              sm->translation_buckets, 0);
2930       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
2931                                           format_ed_session_kvp);
2932
2933
2934       nat_affinity_enable ();
2935
2936       nat_ha_enable (nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb, nat_ha_sref_ed_cb);
2937     }
2938   else
2939     {
2940       sm->worker_out2in_cb = snat_get_worker_out2in_cb;
2941       sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2942       sm->out2in_node_index = sm->ei_out2in_node_index;
2943       sm->in2out_node_index = sm->ei_in2out_node_index;
2944       sm->in2out_output_node_index = sm->ei_in2out_output_node_index;
2945       sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
2946       sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
2947
2948       nat_ha_enable (nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
2949     }
2950
2951   // c.static_mapping & c.connection_tracking require
2952   // session database
2953   if (!c.static_mapping_only
2954       || (c.static_mapping_only && c.connection_tracking))
2955     {
2956       snat_main_per_thread_data_t *tsm;
2957       /* *INDENT-OFF* */
2958       vec_foreach (tsm, sm->per_thread_data)
2959         {
2960           nat44_db_init (tsm);
2961         }
2962       /* *INDENT-ON* */
2963     }
2964   else
2965     {
2966       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2967       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2968     }
2969
2970   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
2971                         "static_mapping_by_local", static_mapping_buckets,
2972                         static_mapping_memory_size);
2973   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
2974                                      format_static_mapping_kvp);
2975
2976   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
2977                         "static_mapping_by_external",
2978                         static_mapping_buckets, static_mapping_memory_size);
2979   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
2980                                      format_static_mapping_kvp);
2981
2982   // last: reset counters
2983   vlib_zero_simple_counter (&sm->total_users, 0);
2984   vlib_zero_simple_counter (&sm->total_sessions, 0);
2985   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2986
2987   sm->enabled = 1;
2988   sm->rconfig = c;
2989
2990   return 0;
2991 }
2992
2993 void
2994 nat44_addresses_free (snat_address_t ** addresses)
2995 {
2996   snat_address_t *ap;
2997   /* *INDENT-OFF* */
2998   vec_foreach (ap, *addresses)
2999     {
3000     #define _(N, i, n, s) \
3001       vec_free (ap->busy_##n##_ports_per_thread);
3002       foreach_nat_protocol
3003     #undef _
3004     }
3005   /* *INDENT-ON* */
3006   vec_free (*addresses);
3007   *addresses = 0;
3008 }
3009
3010 int
3011 nat44_plugin_disable ()
3012 {
3013   snat_main_t *sm = &snat_main;
3014   snat_interface_t *i, *vec;
3015   int error = 0;
3016
3017   if (!sm->enabled)
3018     {
3019       nat_log_err ("nat44 is disabled");
3020       return 1;
3021     }
3022
3023   // first unregister all nodes from interfaces
3024   vec = vec_dup (sm->interfaces);
3025   /* *INDENT-OFF* */
3026   vec_foreach (i, vec)
3027     {
3028       if (nat_interface_is_inside(i))
3029         error = snat_interface_add_del (i->sw_if_index, 1, 1);
3030       if (nat_interface_is_outside(i))
3031         error = snat_interface_add_del (i->sw_if_index, 0, 1);
3032
3033       if (error)
3034         {
3035           nat_log_err ("error occurred while removing interface %u",
3036                        i->sw_if_index);
3037         }
3038     }
3039   /* *INDENT-ON* */
3040   vec_free (vec);
3041   sm->interfaces = 0;
3042
3043   vec = vec_dup (sm->output_feature_interfaces);
3044   /* *INDENT-OFF* */
3045   vec_foreach (i, vec)
3046     {
3047       if (nat_interface_is_inside(i))
3048         error = snat_interface_add_del_output_feature (i->sw_if_index, 1, 1);
3049       if (nat_interface_is_outside(i))
3050         error = snat_interface_add_del_output_feature (i->sw_if_index, 0, 1);
3051
3052       if (error)
3053         {
3054           nat_log_err ("error occurred while removing interface %u",
3055                        i->sw_if_index);
3056         }
3057     }
3058   /* *INDENT-ON* */
3059   vec_free (vec);
3060   sm->output_feature_interfaces = 0;
3061
3062   vec_free (sm->max_translations_per_fib);
3063
3064   if (sm->endpoint_dependent)
3065     {
3066       nat_affinity_disable ();
3067       clib_bihash_free_16_8 (&sm->out2in_ed);
3068     }
3069
3070   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
3071   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
3072
3073   if (!sm->static_mapping_only ||
3074       (sm->static_mapping_only && sm->static_mapping_connection_tracking))
3075     {
3076       snat_main_per_thread_data_t *tsm;
3077      /* *INDENT-OFF* */
3078       vec_foreach (tsm, sm->per_thread_data)
3079         {
3080           nat44_db_free (tsm);
3081         }
3082       /* *INDENT-ON* */
3083     }
3084
3085   pool_free (sm->static_mappings);
3086
3087   nat44_addresses_free (&sm->addresses);
3088   nat44_addresses_free (&sm->twice_nat_addresses);
3089
3090
3091   vec_free (sm->to_resolve);
3092   vec_free (sm->auto_add_sw_if_indices);
3093   vec_free (sm->auto_add_sw_if_indices_twice_nat);
3094
3095   sm->to_resolve = 0;
3096   sm->auto_add_sw_if_indices = 0;
3097   sm->auto_add_sw_if_indices_twice_nat = 0;
3098
3099   sm->forwarding_enabled = 0;
3100
3101   sm->enabled = 0;
3102   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
3103
3104   return 0;
3105 }
3106
3107 void
3108 snat_free_outside_address_and_port (snat_address_t * addresses,
3109                                     u32 thread_index,
3110                                     ip4_address_t * addr,
3111                                     u16 port, nat_protocol_t protocol)
3112 {
3113   snat_address_t *a;
3114   u32 address_index;
3115   u16 port_host_byte_order = clib_net_to_host_u16 (port);
3116
3117   for (address_index = 0; address_index < vec_len (addresses);
3118        address_index++)
3119     {
3120       if (addresses[address_index].addr.as_u32 == addr->as_u32)
3121         break;
3122     }
3123
3124   ASSERT (address_index < vec_len (addresses));
3125
3126   a = addresses + address_index;
3127
3128   switch (protocol)
3129     {
3130 #define _(N, i, n, s) \
3131     case NAT_PROTOCOL_##N: \
3132       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
3133       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
3134       a->busy_##n##_ports--; \
3135       a->busy_##n##_ports_per_thread[thread_index]--; \
3136       break;
3137       foreach_nat_protocol
3138 #undef _
3139     default:
3140       nat_elog_info ("unknown protocol");
3141       return;
3142     }
3143 }
3144
3145 static int
3146 nat_set_outside_address_and_port (snat_address_t * addresses,
3147                                   u32 thread_index, ip4_address_t addr,
3148                                   u16 port, nat_protocol_t protocol)
3149 {
3150   snat_address_t *a = 0;
3151   u32 address_index;
3152   u16 port_host_byte_order = clib_net_to_host_u16 (port);
3153
3154   for (address_index = 0; address_index < vec_len (addresses);
3155        address_index++)
3156     {
3157       if (addresses[address_index].addr.as_u32 != addr.as_u32)
3158         continue;
3159
3160       a = addresses + address_index;
3161       switch (protocol)
3162         {
3163 #define _(N, j, n, s) \
3164         case NAT_PROTOCOL_##N: \
3165           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
3166             return VNET_API_ERROR_INSTANCE_IN_USE; \
3167           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
3168           a->busy_##n##_ports_per_thread[thread_index]++; \
3169           a->busy_##n##_ports++; \
3170           return 0;
3171           foreach_nat_protocol
3172 #undef _
3173         default:
3174           nat_elog_info ("unknown protocol");
3175           return 1;
3176         }
3177     }
3178
3179   return VNET_API_ERROR_NO_SUCH_ENTRY;
3180 }
3181
3182 int
3183 snat_static_mapping_match (snat_main_t * sm,
3184                            ip4_address_t match_addr,
3185                            u16 match_port,
3186                            u32 match_fib_index,
3187                            nat_protocol_t match_protocol,
3188                            ip4_address_t * mapping_addr,
3189                            u16 * mapping_port,
3190                            u32 * mapping_fib_index,
3191                            u8 by_external,
3192                            u8 * is_addr_only,
3193                            twice_nat_type_t * twice_nat,
3194                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
3195                            u8 * is_identity_nat, snat_static_mapping_t ** out)
3196 {
3197   clib_bihash_kv_8_8_t kv, value;
3198   clib_bihash_8_8_t *mapping_hash;
3199   snat_static_mapping_t *m;
3200   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
3201   nat44_lb_addr_port_t *local;
3202   u8 backend_index;
3203
3204   if (!by_external)
3205     {
3206       mapping_hash = &sm->static_mapping_by_local;
3207       init_nat_k (&kv, match_addr, match_port, match_fib_index,
3208                   match_protocol);
3209       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3210         {
3211           /* Try address only mapping */
3212           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
3213           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3214             return 1;
3215         }
3216     }
3217   else
3218     {
3219       mapping_hash = &sm->static_mapping_by_external;
3220       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
3221       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3222         {
3223           /* Try address only mapping */
3224           init_nat_k (&kv, match_addr, 0, 0, 0);
3225           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3226             return 1;
3227         }
3228     }
3229
3230   m = pool_elt_at_index (sm->static_mappings, value.value);
3231
3232   if (by_external)
3233     {
3234       if (is_lb_static_mapping (m))
3235         {
3236           if (PREDICT_FALSE (lb != 0))
3237             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
3238           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
3239                                                           match_addr,
3240                                                           match_protocol,
3241                                                           match_port,
3242                                                           &backend_index))
3243             {
3244               local = pool_elt_at_index (m->locals, backend_index);
3245               *mapping_addr = local->addr;
3246               *mapping_port = local->port;
3247               *mapping_fib_index = local->fib_index;
3248               goto end;
3249             }
3250           // pick locals matching this worker
3251           if (PREDICT_FALSE (sm->num_workers > 1))
3252             {
3253               u32 thread_index = vlib_get_thread_index ();
3254               /* *INDENT-OFF* */
3255               pool_foreach_index (i, m->locals,
3256               ({
3257                 local = pool_elt_at_index (m->locals, i);
3258
3259                 ip4_header_t ip = {
3260                   .src_address = local->addr,
3261                 };
3262
3263                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
3264                     thread_index)
3265                   {
3266                     vec_add1 (tmp, i);
3267                   }
3268               }));
3269               /* *INDENT-ON* */
3270               ASSERT (vec_len (tmp) != 0);
3271             }
3272           else
3273             {
3274               /* *INDENT-OFF* */
3275               pool_foreach_index (i, m->locals,
3276               ({
3277                 vec_add1 (tmp, i);
3278               }));
3279               /* *INDENT-ON* */
3280             }
3281           hi = vec_len (tmp) - 1;
3282           local = pool_elt_at_index (m->locals, tmp[hi]);
3283           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
3284           while (lo < hi)
3285             {
3286               mid = ((hi - lo) >> 1) + lo;
3287               local = pool_elt_at_index (m->locals, tmp[mid]);
3288               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
3289             }
3290           local = pool_elt_at_index (m->locals, tmp[lo]);
3291           if (!(local->prefix >= rand))
3292             return 1;
3293           *mapping_addr = local->addr;
3294           *mapping_port = local->port;
3295           *mapping_fib_index = local->fib_index;
3296           if (m->affinity)
3297             {
3298               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
3299                                                 match_protocol, match_port,
3300                                                 tmp[lo], m->affinity,
3301                                                 m->affinity_per_service_list_head_index))
3302                 nat_elog_info ("create affinity record failed");
3303             }
3304           vec_free (tmp);
3305         }
3306       else
3307         {
3308           if (PREDICT_FALSE (lb != 0))
3309             *lb = NO_LB_NAT;
3310           *mapping_fib_index = m->fib_index;
3311           *mapping_addr = m->local_addr;
3312           /* Address only mapping doesn't change port */
3313           *mapping_port = is_addr_only_static_mapping (m) ? match_port
3314             : m->local_port;
3315         }
3316     }
3317   else
3318     {
3319       *mapping_addr = m->external_addr;
3320       /* Address only mapping doesn't change port */
3321       *mapping_port = is_addr_only_static_mapping (m) ? match_port
3322         : m->external_port;
3323       *mapping_fib_index = sm->outside_fib_index;
3324     }
3325
3326 end:
3327   if (PREDICT_FALSE (is_addr_only != 0))
3328     *is_addr_only = is_addr_only_static_mapping (m);
3329
3330   if (PREDICT_FALSE (twice_nat != 0))
3331     *twice_nat = m->twice_nat;
3332
3333   if (PREDICT_FALSE (is_identity_nat != 0))
3334     *is_identity_nat = is_identity_static_mapping (m);
3335
3336   if (out != 0)
3337     *out = m;
3338
3339   return 0;
3340 }
3341
3342 int
3343 snat_alloc_outside_address_and_port (snat_address_t * addresses,
3344                                      u32 fib_index,
3345                                      u32 thread_index,
3346                                      nat_protocol_t proto,
3347                                      ip4_address_t * addr,
3348                                      u16 * port,
3349                                      u16 port_per_thread,
3350                                      u32 snat_thread_index)
3351 {
3352   snat_main_t *sm = &snat_main;
3353
3354   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, proto,
3355                                   addr, port, port_per_thread,
3356                                   snat_thread_index);
3357 }
3358
3359 static int
3360 nat_alloc_addr_and_port_default (snat_address_t * addresses,
3361                                  u32 fib_index,
3362                                  u32 thread_index,
3363                                  nat_protocol_t proto,
3364                                  ip4_address_t * addr,
3365                                  u16 * port,
3366                                  u16 port_per_thread, u32 snat_thread_index)
3367 {
3368   int i;
3369   snat_address_t *a, *ga = 0;
3370   u32 portnum;
3371
3372   for (i = 0; i < vec_len (addresses); i++)
3373     {
3374       a = addresses + i;
3375       switch (proto)
3376         {
3377 #define _(N, j, n, s) \
3378         case NAT_PROTOCOL_##N: \
3379           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
3380             { \
3381               if (a->fib_index == fib_index) \
3382                 { \
3383                   while (1) \
3384                     { \
3385                       portnum = (port_per_thread * \
3386                         snat_thread_index) + \
3387                         snat_random_port(0, port_per_thread - 1) + 1024; \
3388                       if (a->busy_##n##_port_refcounts[portnum]) \
3389                         continue; \
3390                       --a->busy_##n##_port_refcounts[portnum]; \
3391                       a->busy_##n##_ports_per_thread[thread_index]++; \
3392                       a->busy_##n##_ports++; \
3393                       *addr = a->addr; \
3394                       *port = clib_host_to_net_u16(portnum); \
3395                       return 0; \
3396                     } \
3397                 } \
3398               else if (a->fib_index == ~0) \
3399                 { \
3400                   ga = a; \
3401                 } \
3402             } \
3403           break;
3404           foreach_nat_protocol
3405 #undef _
3406         default:
3407           nat_elog_info ("unknown protocol");
3408           return 1;
3409         }
3410
3411     }
3412
3413   if (ga)
3414     {
3415       a = ga;
3416       switch (proto)
3417         {
3418 #define _(N, j, n, s) \
3419         case NAT_PROTOCOL_##N: \
3420           while (1) \
3421             { \
3422               portnum = (port_per_thread * \
3423                 snat_thread_index) + \
3424                 snat_random_port(0, port_per_thread - 1) + 1024; \
3425               if (a->busy_##n##_port_refcounts[portnum]) \
3426                 continue; \
3427               ++a->busy_##n##_port_refcounts[portnum]; \
3428               a->busy_##n##_ports_per_thread[thread_index]++; \
3429               a->busy_##n##_ports++; \
3430               *addr = a->addr; \
3431               *port = clib_host_to_net_u16(portnum); \
3432               return 0; \
3433             }
3434           break;
3435           foreach_nat_protocol
3436 #undef _
3437         default:
3438           nat_elog_info ("unknown protocol");
3439           return 1;
3440         }
3441     }
3442
3443   /* Totally out of translations to use... */
3444   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
3445   return 1;
3446 }
3447
3448 static int
3449 nat_alloc_addr_and_port_mape (snat_address_t * addresses, u32 fib_index,
3450                               u32 thread_index, nat_protocol_t proto,
3451                               ip4_address_t * addr, u16 * port,
3452                               u16 port_per_thread, u32 snat_thread_index)
3453 {
3454   snat_main_t *sm = &snat_main;
3455   snat_address_t *a = addresses;
3456   u16 m, ports, portnum, A, j;
3457   m = 16 - (sm->psid_offset + sm->psid_length);
3458   ports = (1 << (16 - sm->psid_length)) - (1 << m);
3459
3460   if (!vec_len (addresses))
3461     goto exhausted;
3462
3463   switch (proto)
3464     {
3465 #define _(N, i, n, s) \
3466     case NAT_PROTOCOL_##N: \
3467       if (a->busy_##n##_ports < ports) \
3468         { \
3469           while (1) \
3470             { \
3471               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3472               j = snat_random_port(0, pow2_mask(m)); \
3473               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3474               if (a->busy_##n##_port_refcounts[portnum]) \
3475                 continue; \
3476               ++a->busy_##n##_port_refcounts[portnum]; \
3477               a->busy_##n##_ports++; \
3478               *addr = a->addr; \
3479               *port = clib_host_to_net_u16 (portnum); \
3480               return 0; \
3481             } \
3482         } \
3483       break;
3484       foreach_nat_protocol
3485 #undef _
3486     default:
3487       nat_elog_info ("unknown protocol");
3488       return 1;
3489     }
3490
3491 exhausted:
3492   /* Totally out of translations to use... */
3493   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
3494   return 1;
3495 }
3496
3497 static int
3498 nat_alloc_addr_and_port_range (snat_address_t * addresses, u32 fib_index,
3499                                u32 thread_index, nat_protocol_t proto,
3500                                ip4_address_t * addr, u16 * port,
3501                                u16 port_per_thread, u32 snat_thread_index)
3502 {
3503   snat_main_t *sm = &snat_main;
3504   snat_address_t *a = addresses;
3505   u16 portnum, ports;
3506
3507   ports = sm->end_port - sm->start_port + 1;
3508
3509   if (!vec_len (addresses))
3510     goto exhausted;
3511
3512   switch (proto)
3513     {
3514 #define _(N, i, n, s) \
3515     case NAT_PROTOCOL_##N: \
3516       if (a->busy_##n##_ports < ports) \
3517         { \
3518           while (1) \
3519             { \
3520               portnum = snat_random_port(sm->start_port, sm->end_port); \
3521               if (a->busy_##n##_port_refcounts[portnum]) \
3522                 continue; \
3523               ++a->busy_##n##_port_refcounts[portnum]; \
3524               a->busy_##n##_ports++; \
3525               *addr = a->addr; \
3526               *port = clib_host_to_net_u16 (portnum); \
3527               return 0; \
3528             } \
3529         } \
3530       break;
3531       foreach_nat_protocol
3532 #undef _
3533     default:
3534       nat_elog_info ("unknown protocol");
3535       return 1;
3536     }
3537
3538 exhausted:
3539   /* Totally out of translations to use... */
3540   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
3541   return 1;
3542 }
3543
3544 void
3545 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3546 {
3547   snat_main_t *sm = &snat_main;
3548   dpo_id_t dpo_v4 = DPO_INVALID;
3549   fib_prefix_t pfx = {
3550     .fp_proto = FIB_PROTOCOL_IP4,
3551     .fp_len = 32,
3552     .fp_addr.ip4.as_u32 = addr.as_u32,
3553   };
3554
3555   if (is_add)
3556     {
3557       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3558       fib_table_entry_special_dpo_add (0, &pfx, sm->fib_src_hi,
3559                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3560       dpo_reset (&dpo_v4);
3561     }
3562   else
3563     {
3564       fib_table_entry_special_remove (0, &pfx, sm->fib_src_hi);
3565     }
3566 }
3567
3568 static u32
3569 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3570                            u8 is_output)
3571 {
3572   snat_main_t *sm = &snat_main;
3573   u32 next_worker_index = 0;
3574   u32 hash;
3575
3576   next_worker_index = sm->first_worker_index;
3577   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3578     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3579
3580   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3581     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3582   else
3583     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3584
3585   return next_worker_index;
3586 }
3587
3588 static u32
3589 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3590                            u32 rx_fib_index0, u8 is_output)
3591 {
3592   snat_main_t *sm = &snat_main;
3593   udp_header_t *udp;
3594   u16 port;
3595   clib_bihash_kv_8_8_t kv, value;
3596   snat_static_mapping_t *m;
3597   u32 proto;
3598   u32 next_worker_index = 0;
3599
3600   /* first try static mappings without port */
3601   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3602     {
3603       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
3604       if (!clib_bihash_search_8_8
3605           (&sm->static_mapping_by_external, &kv, &value))
3606         {
3607           m = pool_elt_at_index (sm->static_mappings, value.value);
3608           return m->workers[0];
3609         }
3610     }
3611
3612   proto = ip_proto_to_nat_proto (ip0->protocol);
3613   udp = ip4_next_header (ip0);
3614   port = udp->dst_port;
3615
3616   /* unknown protocol */
3617   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3618     {
3619       /* use current thread */
3620       return vlib_get_thread_index ();
3621     }
3622
3623   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3624     {
3625       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3626       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3627       if (!icmp_type_is_error_message
3628           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3629         port = vnet_buffer (b)->ip.reass.l4_src_port;
3630       else
3631         {
3632           /* if error message, then it's not fragmented and we can access it */
3633           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3634           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3635           void *l4_header = ip4_next_header (inner_ip);
3636           switch (proto)
3637             {
3638             case NAT_PROTOCOL_ICMP:
3639               icmp = (icmp46_header_t *) l4_header;
3640               echo = (icmp_echo_header_t *) (icmp + 1);
3641               port = echo->identifier;
3642               break;
3643             case NAT_PROTOCOL_UDP:
3644             case NAT_PROTOCOL_TCP:
3645               port = ((tcp_udp_header_t *) l4_header)->src_port;
3646               break;
3647             default:
3648               return vlib_get_thread_index ();
3649             }
3650         }
3651     }
3652
3653   /* try static mappings with port */
3654   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3655     {
3656       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
3657       if (!clib_bihash_search_8_8
3658           (&sm->static_mapping_by_external, &kv, &value))
3659         {
3660           m = pool_elt_at_index (sm->static_mappings, value.value);
3661           return m->workers[0];
3662         }
3663     }
3664
3665   /* worker by outside port */
3666   next_worker_index = sm->first_worker_index;
3667   next_worker_index +=
3668     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3669   return next_worker_index;
3670 }
3671
3672 static u32
3673 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3674                                u8 is_output)
3675 {
3676   snat_main_t *sm = &snat_main;
3677   u32 next_worker_index = sm->first_worker_index;
3678   u32 hash;
3679
3680   clib_bihash_kv_16_8_t kv16, value16;
3681   snat_main_per_thread_data_t *tsm;
3682   udp_header_t *udp;
3683
3684   if (PREDICT_FALSE (is_output))
3685     {
3686       u32 fib_index = sm->outside_fib_index;
3687       nat_outside_fib_t *outside_fib;
3688       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3689       fib_prefix_t pfx = {
3690         .fp_proto = FIB_PROTOCOL_IP4,
3691         .fp_len = 32,
3692         .fp_addr = {
3693                     .ip4.as_u32 = ip->dst_address.as_u32,
3694                     }
3695         ,
3696       };
3697
3698       udp = ip4_next_header (ip);
3699
3700       switch (vec_len (sm->outside_fibs))
3701         {
3702         case 0:
3703           fib_index = sm->outside_fib_index;
3704           break;
3705         case 1:
3706           fib_index = sm->outside_fibs[0].fib_index;
3707           break;
3708         default:
3709             /* *INDENT-OFF* */
3710             vec_foreach (outside_fib, sm->outside_fibs)
3711               {
3712                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3713                 if (FIB_NODE_INDEX_INVALID != fei)
3714                   {
3715                     if (fib_entry_get_resolving_interface (fei) != ~0)
3716                       {
3717                         fib_index = outside_fib->fib_index;
3718                         break;
3719                       }
3720                   }
3721               }
3722             /* *INDENT-ON* */
3723           break;
3724         }
3725
3726       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3727                  udp->dst_port, fib_index, ip->protocol);
3728
3729       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3730                                                   &kv16, &value16)))
3731         {
3732           tsm =
3733             vec_elt_at_index (sm->per_thread_data,
3734                               ed_value_get_thread_index (&value16));
3735           next_worker_index += tsm->thread_index;
3736
3737           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3738                                   next_worker_index, fib_index,
3739                                   clib_net_to_host_u32 (ip->
3740                                                         src_address.as_u32),
3741                                   clib_net_to_host_u32 (ip->
3742                                                         dst_address.as_u32));
3743
3744           return next_worker_index;
3745         }
3746     }
3747
3748   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3749     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3750
3751   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3752     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3753   else
3754     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3755
3756   if (PREDICT_TRUE (!is_output))
3757     {
3758       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3759                               next_worker_index, rx_fib_index,
3760                               clib_net_to_host_u32 (ip->src_address.as_u32),
3761                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3762     }
3763   else
3764     {
3765       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3766                               next_worker_index, rx_fib_index,
3767                               clib_net_to_host_u32 (ip->src_address.as_u32),
3768                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3769     }
3770
3771   return next_worker_index;
3772 }
3773
3774 static u32
3775 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3776                                u32 rx_fib_index, u8 is_output)
3777 {
3778   snat_main_t *sm = &snat_main;
3779   clib_bihash_kv_8_8_t kv, value;
3780   clib_bihash_kv_16_8_t kv16, value16;
3781   snat_main_per_thread_data_t *tsm;
3782
3783   u32 proto, next_worker_index = 0;
3784   udp_header_t *udp;
3785   u16 port;
3786   snat_static_mapping_t *m;
3787   u32 hash;
3788
3789   proto = ip_proto_to_nat_proto (ip->protocol);
3790
3791   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3792     {
3793       udp = ip4_next_header (ip);
3794
3795       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3796                  udp->src_port, rx_fib_index, ip->protocol);
3797
3798       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3799                                                   &kv16, &value16)))
3800         {
3801           tsm =
3802             vec_elt_at_index (sm->per_thread_data,
3803                               ed_value_get_thread_index (&value16));
3804           vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
3805             ed_value_get_session_index (&value16);
3806           next_worker_index = sm->first_worker_index + tsm->thread_index;
3807           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3808                                   next_worker_index, rx_fib_index,
3809                                   clib_net_to_host_u32 (ip->
3810                                                         src_address.as_u32),
3811                                   clib_net_to_host_u32 (ip->
3812                                                         dst_address.as_u32));
3813           return next_worker_index;
3814         }
3815     }
3816   else if (proto == NAT_PROTOCOL_ICMP)
3817     {
3818       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
3819         {
3820           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3821                                                       &kv16, &value16)))
3822             {
3823               tsm =
3824                 vec_elt_at_index (sm->per_thread_data,
3825                                   ed_value_get_thread_index (&value16));
3826               next_worker_index = sm->first_worker_index + tsm->thread_index;
3827               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3828                                       next_worker_index, rx_fib_index,
3829                                       clib_net_to_host_u32 (ip->
3830                                                             src_address.as_u32),
3831                                       clib_net_to_host_u32 (ip->
3832                                                             dst_address.as_u32));
3833               return next_worker_index;
3834             }
3835         }
3836     }
3837
3838   /* first try static mappings without port */
3839   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3840     {
3841       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3842       if (!clib_bihash_search_8_8
3843           (&sm->static_mapping_by_external, &kv, &value))
3844         {
3845           m = pool_elt_at_index (sm->static_mappings, value.value);
3846           next_worker_index = m->workers[0];
3847           goto done;
3848         }
3849     }
3850
3851   /* unknown protocol */
3852   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3853     {
3854       /* use current thread */
3855       next_worker_index = vlib_get_thread_index ();
3856       goto done;
3857     }
3858
3859   udp = ip4_next_header (ip);
3860   port = udp->dst_port;
3861
3862   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3863     {
3864       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3865       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3866       if (!icmp_type_is_error_message
3867           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3868         port = vnet_buffer (b)->ip.reass.l4_src_port;
3869       else
3870         {
3871           /* if error message, then it's not fragmented and we can access it */
3872           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3873           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3874           void *l4_header = ip4_next_header (inner_ip);
3875           switch (proto)
3876             {
3877             case NAT_PROTOCOL_ICMP:
3878               icmp = (icmp46_header_t *) l4_header;
3879               echo = (icmp_echo_header_t *) (icmp + 1);
3880               port = echo->identifier;
3881               break;
3882             case NAT_PROTOCOL_UDP:
3883             case NAT_PROTOCOL_TCP:
3884               port = ((tcp_udp_header_t *) l4_header)->src_port;
3885               break;
3886             default:
3887               next_worker_index = vlib_get_thread_index ();
3888               goto done;
3889             }
3890         }
3891     }
3892
3893   /* try static mappings with port */
3894   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3895     {
3896       init_nat_k (&kv, ip->dst_address, port, 0, proto);
3897       if (!clib_bihash_search_8_8
3898           (&sm->static_mapping_by_external, &kv, &value))
3899         {
3900           m = pool_elt_at_index (sm->static_mappings, value.value);
3901           if (!is_lb_static_mapping (m))
3902             {
3903               next_worker_index = m->workers[0];
3904               goto done;
3905             }
3906
3907           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3908             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3909
3910           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3911             next_worker_index =
3912               m->workers[hash & (_vec_len (m->workers) - 1)];
3913           else
3914             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3915           goto done;
3916         }
3917     }
3918
3919   /* worker by outside port */
3920   next_worker_index = sm->first_worker_index;
3921   next_worker_index +=
3922     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3923
3924 done:
3925   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3926                           clib_net_to_host_u32 (ip->src_address.as_u32),
3927                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3928   return next_worker_index;
3929 }
3930
3931 void
3932 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3933                 ip4_address_t * out_addr, u16 out_port,
3934                 ip4_address_t * eh_addr, u16 eh_port,
3935                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3936                 u32 fib_index, u16 flags, u32 thread_index)
3937 {
3938   snat_main_t *sm = &snat_main;
3939   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3940   snat_user_t *u;
3941   snat_session_t *s;
3942   clib_bihash_kv_8_8_t kv;
3943   vlib_main_t *vm = vlib_get_main ();
3944   f64 now = vlib_time_now (vm);
3945   nat_outside_fib_t *outside_fib;
3946   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3947   fib_prefix_t pfx = {
3948     .fp_proto = FIB_PROTOCOL_IP4,
3949     .fp_len = 32,
3950     .fp_addr = {
3951                 .ip4.as_u32 = eh_addr->as_u32,
3952                 },
3953   };
3954
3955   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3956     {
3957       if (nat_set_outside_address_and_port
3958           (sm->addresses, thread_index, *out_addr, out_port, proto))
3959         return;
3960     }
3961
3962   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3963   if (!u)
3964     return;
3965
3966   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3967   if (!s)
3968     return;
3969
3970   if (sm->endpoint_dependent)
3971     {
3972       nat_ed_lru_insert (tsm, s, now, nat_proto_to_ip_proto (proto));
3973     }
3974
3975   s->out2in.addr.as_u32 = out_addr->as_u32;
3976   s->out2in.port = out_port;
3977   s->nat_proto = proto;
3978   s->last_heard = now;
3979   s->flags = flags;
3980   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3981   s->ext_host_port = eh_port;
3982   user_session_increment (sm, u, snat_is_session_static (s));
3983   switch (vec_len (sm->outside_fibs))
3984     {
3985     case 0:
3986       s->out2in.fib_index = sm->outside_fib_index;
3987       break;
3988     case 1:
3989       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3990       break;
3991     default:
3992       /* *INDENT-OFF* */
3993       vec_foreach (outside_fib, sm->outside_fibs)
3994         {
3995           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3996           if (FIB_NODE_INDEX_INVALID != fei)
3997             {
3998               if (fib_entry_get_resolving_interface (fei) != ~0)
3999                 {
4000                   s->out2in.fib_index = outside_fib->fib_index;
4001                   break;
4002                 }
4003             }
4004         }
4005       /* *INDENT-ON* */
4006       break;
4007     }
4008   init_nat_o2i_kv (&kv, s, s - tsm->sessions);
4009   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
4010     nat_elog_warn ("out2in key add failed");
4011
4012   s->in2out.addr.as_u32 = in_addr->as_u32;
4013   s->in2out.port = in_port;
4014   s->in2out.fib_index = fib_index;
4015   init_nat_i2o_kv (&kv, s, s - tsm->sessions);
4016   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
4017     nat_elog_warn ("in2out key add failed");
4018 }
4019
4020 void
4021 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
4022                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
4023                 u32 ti)
4024 {
4025   snat_main_t *sm = &snat_main;
4026   clib_bihash_kv_8_8_t kv, value;
4027   u32 thread_index;
4028   snat_session_t *s;
4029   snat_main_per_thread_data_t *tsm;
4030
4031   if (sm->num_workers > 1)
4032     thread_index =
4033       sm->first_worker_index +
4034       (sm->workers[(clib_net_to_host_u16 (out_port) -
4035                     1024) / sm->port_per_thread]);
4036   else
4037     thread_index = sm->num_workers;
4038   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
4039
4040   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
4041   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
4042     return;
4043
4044   s = pool_elt_at_index (tsm->sessions, value.value);
4045   nat_free_session_data (sm, s, thread_index, 1);
4046   nat44_delete_session (sm, s, thread_index);
4047 }
4048
4049 void
4050 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
4051                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
4052                 u32 total_pkts, u64 total_bytes, u32 thread_index)
4053 {
4054   snat_main_t *sm = &snat_main;
4055   clib_bihash_kv_8_8_t kv, value;
4056   snat_session_t *s;
4057   snat_main_per_thread_data_t *tsm;
4058
4059   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
4060
4061   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
4062   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
4063     return;
4064
4065   s = pool_elt_at_index (tsm->sessions, value.value);
4066   s->total_pkts = total_pkts;
4067   s->total_bytes = total_bytes;
4068 }
4069
4070 void
4071 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
4072                    ip4_address_t * out_addr, u16 out_port,
4073                    ip4_address_t * eh_addr, u16 eh_port,
4074                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
4075                    u32 fib_index, u16 flags, u32 thread_index)
4076 {
4077   snat_main_t *sm = &snat_main;
4078   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
4079   snat_session_t *s;
4080   clib_bihash_kv_16_8_t kv;
4081   vlib_main_t *vm = vlib_get_main ();
4082   f64 now = vlib_time_now (vm);
4083   nat_outside_fib_t *outside_fib;
4084   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
4085   fib_prefix_t pfx = {
4086     .fp_proto = FIB_PROTOCOL_IP4,
4087     .fp_len = 32,
4088     .fp_addr = {
4089                 .ip4.as_u32 = eh_addr->as_u32,
4090                 },
4091   };
4092
4093
4094   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
4095     {
4096       if (nat_set_outside_address_and_port
4097           (sm->addresses, thread_index, *out_addr, out_port, proto))
4098         return;
4099     }
4100
4101   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
4102     {
4103       if (nat_set_outside_address_and_port
4104           (sm->addresses, thread_index, *ehn_addr, ehn_port, proto))
4105         return;
4106     }
4107
4108   s = nat_ed_session_alloc (sm, thread_index, now, proto);
4109   if (!s)
4110     return;
4111
4112   s->last_heard = now;
4113   s->flags = flags;
4114   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
4115   s->ext_host_nat_port = s->ext_host_port = eh_port;
4116   if (is_twice_nat_session (s))
4117     {
4118       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
4119       s->ext_host_nat_port = ehn_port;
4120     }
4121   switch (vec_len (sm->outside_fibs))
4122     {
4123     case 0:
4124       s->out2in.fib_index = sm->outside_fib_index;
4125       break;
4126     case 1:
4127       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
4128       break;
4129     default:
4130       /* *INDENT-OFF* */
4131       vec_foreach (outside_fib, sm->outside_fibs)
4132         {
4133           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
4134           if (FIB_NODE_INDEX_INVALID != fei)
4135             {
4136               if (fib_entry_get_resolving_interface (fei) != ~0)
4137                 {
4138                   s->out2in.fib_index = outside_fib->fib_index;
4139                   break;
4140                 }
4141             }
4142         }
4143       /* *INDENT-ON* */
4144       break;
4145     }
4146   s->nat_proto = proto;
4147   s->out2in.addr.as_u32 = out_addr->as_u32;
4148   s->out2in.port = out_port;
4149
4150   s->in2out.addr.as_u32 = in_addr->as_u32;
4151   s->in2out.port = in_port;
4152   s->in2out.fib_index = fib_index;
4153
4154   init_ed_kv (&kv, *in_addr, in_port, s->ext_host_nat_addr,
4155               s->ext_host_nat_port, fib_index, nat_proto_to_ip_proto (proto),
4156               thread_index, s - tsm->sessions);
4157   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
4158     nat_elog_warn ("in2out key add failed");
4159
4160   init_ed_kv (&kv, *out_addr, out_port, *eh_addr, eh_port,
4161               s->out2in.fib_index, nat_proto_to_ip_proto (proto),
4162               thread_index, s - tsm->sessions);
4163   if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 1))
4164     nat_elog_warn ("out2in key add failed");
4165 }
4166
4167 void
4168 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
4169                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4170                    u32 fib_index, u32 ti)
4171 {
4172   snat_main_t *sm = &snat_main;
4173   clib_bihash_kv_16_8_t kv, value;
4174   u32 thread_index;
4175   snat_session_t *s;
4176   snat_main_per_thread_data_t *tsm;
4177
4178   if (sm->num_workers > 1)
4179     thread_index =
4180       sm->first_worker_index +
4181       (sm->workers[(clib_net_to_host_u16 (out_port) -
4182                     1024) / sm->port_per_thread]);
4183   else
4184     thread_index = sm->num_workers;
4185   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
4186
4187   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
4188   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
4189     return;
4190
4191   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
4192   nat_free_session_data (sm, s, thread_index, 1);
4193   nat44_delete_session (sm, s, thread_index);
4194 }
4195
4196 void
4197 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
4198                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4199                    u32 fib_index, u32 total_pkts, u64 total_bytes,
4200                    u32 thread_index)
4201 {
4202   snat_main_t *sm = &snat_main;
4203   clib_bihash_kv_16_8_t kv, value;
4204   snat_session_t *s;
4205   snat_main_per_thread_data_t *tsm;
4206
4207   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
4208
4209   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
4210   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
4211     return;
4212
4213   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
4214   s->total_pkts = total_pkts;
4215   s->total_bytes = total_bytes;
4216 }
4217
4218 static u32
4219 nat_calc_bihash_buckets (u32 n_elts)
4220 {
4221   n_elts = n_elts / 2.5;
4222   u64 lower_pow2 = 1;
4223   while (lower_pow2 * 2 < n_elts)
4224     {
4225       lower_pow2 = 2 * lower_pow2;
4226     }
4227   u64 upper_pow2 = 2 * lower_pow2;
4228   if ((upper_pow2 - n_elts) < (n_elts - lower_pow2))
4229     {
4230       if (upper_pow2 <= UINT32_MAX)
4231         {
4232           return upper_pow2;
4233         }
4234     }
4235   return lower_pow2;
4236 }
4237
4238 u32
4239 nat44_get_max_session_limit ()
4240 {
4241   snat_main_t *sm = &snat_main;
4242   u32 max_limit = 0, len = 0;
4243
4244   for (; len < vec_len (sm->max_translations_per_fib); len++)
4245     {
4246       if (max_limit < sm->max_translations_per_fib[len])
4247         max_limit = sm->max_translations_per_fib[len];
4248     }
4249   return max_limit;
4250 }
4251
4252 int
4253 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
4254 {
4255   snat_main_t *sm = &snat_main;
4256   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4257   u32 len = vec_len (sm->max_translations_per_fib);
4258
4259   if (len <= fib_index)
4260     {
4261       vec_validate (sm->max_translations_per_fib, fib_index + 1);
4262
4263       for (; len < vec_len (sm->max_translations_per_fib); len++)
4264         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
4265     }
4266
4267   sm->max_translations_per_fib[fib_index] = session_limit;
4268   return 0;
4269 }
4270
4271 int
4272 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
4273 {
4274   snat_main_t *sm = &snat_main;
4275
4276   if (nat44_set_session_limit (session_limit, vrf_id))
4277     return 1;
4278   sm->max_translations_per_thread = nat44_get_max_session_limit ();
4279
4280   sm->translation_buckets =
4281     nat_calc_bihash_buckets (sm->max_translations_per_thread);
4282
4283   nat44_sessions_clear ();
4284   return 0;
4285 }
4286
4287 void
4288 nat44_db_init (snat_main_per_thread_data_t * tsm)
4289 {
4290   snat_main_t *sm = &snat_main;
4291
4292   pool_alloc (tsm->sessions, sm->max_translations_per_thread);
4293   pool_alloc (tsm->lru_pool, sm->max_translations_per_thread);
4294
4295   dlist_elt_t *head;
4296
4297   pool_get (tsm->lru_pool, head);
4298   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
4299   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
4300
4301   pool_get (tsm->lru_pool, head);
4302   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
4303   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
4304
4305   pool_get (tsm->lru_pool, head);
4306   tsm->udp_lru_head_index = head - tsm->lru_pool;
4307   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
4308
4309   pool_get (tsm->lru_pool, head);
4310   tsm->icmp_lru_head_index = head - tsm->lru_pool;
4311   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
4312
4313   pool_get (tsm->lru_pool, head);
4314   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
4315   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
4316
4317   if (sm->endpoint_dependent)
4318     {
4319       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
4320                              sm->translation_buckets, 0);
4321       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
4322                                           format_ed_session_kvp);
4323       /*
4324          clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4325          sm->translation_buckets, 0);
4326          clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4327          format_ed_session_kvp); */
4328     }
4329   else
4330     {
4331       clib_bihash_init_8_8 (&tsm->in2out, "in2out", sm->translation_buckets,
4332                             0);
4333       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
4334       clib_bihash_init_8_8 (&tsm->out2in, "out2in", sm->translation_buckets,
4335                             0);
4336       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
4337     }
4338
4339   // TODO: ED nat is not using these
4340   // before removal large refactor required
4341   pool_alloc (tsm->list_pool, sm->max_translations_per_thread);
4342   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets, 0);
4343   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
4344 }
4345
4346 void
4347 nat44_db_free (snat_main_per_thread_data_t * tsm)
4348 {
4349   snat_main_t *sm = &snat_main;
4350
4351   pool_free (tsm->sessions);
4352   pool_free (tsm->lru_pool);
4353
4354   if (sm->endpoint_dependent)
4355     {
4356       clib_bihash_free_16_8 (&tsm->in2out_ed);
4357       vec_free (tsm->per_vrf_sessions_vec);
4358     }
4359   else
4360     {
4361       clib_bihash_free_8_8 (&tsm->in2out);
4362       clib_bihash_free_8_8 (&tsm->out2in);
4363     }
4364
4365   // TODO: resolve static mappings (put only to !ED)
4366   pool_free (tsm->users);
4367   pool_free (tsm->list_pool);
4368   clib_bihash_free_8_8 (&tsm->user_hash);
4369 }
4370
4371 void
4372 nat44_sessions_clear ()
4373 {
4374   snat_main_t *sm = &snat_main;
4375   snat_main_per_thread_data_t *tsm;
4376
4377   if (sm->endpoint_dependent)
4378     {
4379       clib_bihash_free_16_8 (&sm->out2in_ed);
4380       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4381                              clib_max (1,
4382                                        sm->num_workers) *
4383                              sm->translation_buckets, 0);
4384       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4385                                           format_ed_session_kvp);
4386     }
4387
4388   /* *INDENT-OFF* */
4389   vec_foreach (tsm, sm->per_thread_data)
4390     {
4391       u32 ti;
4392
4393       nat44_db_free (tsm);
4394       nat44_db_init (tsm);
4395
4396       ti = tsm->snat_thread_index;
4397       vlib_set_simple_counter (&sm->total_users, ti, 0, 0);
4398       vlib_set_simple_counter (&sm->total_sessions, ti, 0, 0);
4399     }
4400   /* *INDENT-ON* */
4401 }
4402
4403 static void
4404 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4405                                  uword opaque,
4406                                  u32 sw_if_index,
4407                                  ip4_address_t * address,
4408                                  u32 address_length,
4409                                  u32 if_address_index, u32 is_delete)
4410 {
4411   snat_main_t *sm = &snat_main;
4412   snat_static_map_resolve_t *rp;
4413   snat_static_mapping_t *m;
4414   clib_bihash_kv_8_8_t kv, value;
4415   int i, rv;
4416   ip4_address_t l_addr;
4417
4418   if (!sm->enabled)
4419     return;
4420
4421   for (i = 0; i < vec_len (sm->to_resolve); i++)
4422     {
4423       rp = sm->to_resolve + i;
4424       if (rp->addr_only == 0)
4425         continue;
4426       if (rp->sw_if_index == sw_if_index)
4427         goto match;
4428     }
4429
4430   return;
4431
4432 match:
4433   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
4434               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
4435   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4436     m = 0;
4437   else
4438     m = pool_elt_at_index (sm->static_mappings, value.value);
4439
4440   if (!is_delete)
4441     {
4442       /* Don't trip over lease renewal, static config */
4443       if (m)
4444         return;
4445     }
4446   else
4447     {
4448       if (!m)
4449         return;
4450     }
4451
4452   /* Indetity mapping? */
4453   if (rp->l_addr.as_u32 == 0)
4454     l_addr.as_u32 = address[0].as_u32;
4455   else
4456     l_addr.as_u32 = rp->l_addr.as_u32;
4457   /* Add the static mapping */
4458   rv = snat_add_static_mapping (l_addr,
4459                                 address[0],
4460                                 rp->l_port,
4461                                 rp->e_port,
4462                                 rp->vrf_id,
4463                                 rp->addr_only, ~0 /* sw_if_index */ ,
4464                                 rp->proto, !is_delete, rp->twice_nat,
4465                                 rp->out2in_only, rp->tag, rp->identity_nat,
4466                                 rp->pool_addr, rp->exact);
4467   if (rv)
4468     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4469 }
4470
4471 static void
4472 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4473                                        uword opaque,
4474                                        u32 sw_if_index,
4475                                        ip4_address_t * address,
4476                                        u32 address_length,
4477                                        u32 if_address_index, u32 is_delete)
4478 {
4479   snat_main_t *sm = &snat_main;
4480   snat_static_map_resolve_t *rp;
4481   ip4_address_t l_addr;
4482   int i, j;
4483   int rv;
4484   u8 twice_nat = 0;
4485   snat_address_t *addresses = sm->addresses;
4486
4487   if (!sm->enabled)
4488     return;
4489
4490   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4491     {
4492       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4493         goto match;
4494     }
4495
4496   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4497     {
4498       twice_nat = 1;
4499       addresses = sm->twice_nat_addresses;
4500       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4501         goto match;
4502     }
4503
4504   return;
4505
4506 match:
4507   if (!is_delete)
4508     {
4509       /* Don't trip over lease renewal, static config */
4510       for (j = 0; j < vec_len (addresses); j++)
4511         if (addresses[j].addr.as_u32 == address->as_u32)
4512           return;
4513
4514       (void) snat_add_address (sm, address, ~0, twice_nat);
4515       /* Scan static map resolution vector */
4516       for (j = 0; j < vec_len (sm->to_resolve); j++)
4517         {
4518           rp = sm->to_resolve + j;
4519           if (rp->addr_only)
4520             continue;
4521           /* On this interface? */
4522           if (rp->sw_if_index == sw_if_index)
4523             {
4524               /* Indetity mapping? */
4525               if (rp->l_addr.as_u32 == 0)
4526                 l_addr.as_u32 = address[0].as_u32;
4527               else
4528                 l_addr.as_u32 = rp->l_addr.as_u32;
4529               /* Add the static mapping */
4530               rv = snat_add_static_mapping (l_addr,
4531                                             address[0],
4532                                             rp->l_port,
4533                                             rp->e_port,
4534                                             rp->vrf_id,
4535                                             rp->addr_only,
4536                                             ~0 /* sw_if_index */ ,
4537                                             rp->proto,
4538                                             rp->is_add, rp->twice_nat,
4539                                             rp->out2in_only, rp->tag,
4540                                             rp->identity_nat,
4541                                             rp->pool_addr, rp->exact);
4542               if (rv)
4543                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4544                                     "i4", rv);
4545             }
4546         }
4547       return;
4548     }
4549   else
4550     {
4551       (void) snat_del_address (sm, address[0], 1, twice_nat);
4552       return;
4553     }
4554 }
4555
4556 int
4557 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4558                             u8 twice_nat)
4559 {
4560   ip4_main_t *ip4_main = sm->ip4_main;
4561   ip4_address_t *first_int_addr;
4562   snat_static_map_resolve_t *rp;
4563   u32 *indices_to_delete = 0;
4564   int i, j;
4565   u32 *auto_add_sw_if_indices =
4566     twice_nat ? sm->
4567     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4568
4569   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4570     );
4571
4572   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4573     {
4574       if (auto_add_sw_if_indices[i] == sw_if_index)
4575         {
4576           if (is_del)
4577             {
4578               /* if have address remove it */
4579               if (first_int_addr)
4580                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4581               else
4582                 {
4583                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4584                     {
4585                       rp = sm->to_resolve + j;
4586                       if (rp->sw_if_index == sw_if_index)
4587                         vec_add1 (indices_to_delete, j);
4588                     }
4589                   if (vec_len (indices_to_delete))
4590                     {
4591                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4592                         vec_del1 (sm->to_resolve, j);
4593                       vec_free (indices_to_delete);
4594                     }
4595                 }
4596               if (twice_nat)
4597                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4598               else
4599                 vec_del1 (sm->auto_add_sw_if_indices, i);
4600             }
4601           else
4602             return VNET_API_ERROR_VALUE_EXIST;
4603
4604           return 0;
4605         }
4606     }
4607
4608   if (is_del)
4609     return VNET_API_ERROR_NO_SUCH_ENTRY;
4610
4611   /* add to the auto-address list */
4612   if (twice_nat)
4613     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4614   else
4615     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4616
4617   /* If the address is already bound - or static - add it now */
4618   if (first_int_addr)
4619     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4620
4621   return 0;
4622 }
4623
4624 int
4625 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4626                    nat_protocol_t proto, u32 vrf_id, int is_in)
4627 {
4628   snat_main_per_thread_data_t *tsm;
4629   clib_bihash_kv_8_8_t kv, value;
4630   ip4_header_t ip;
4631   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4632   snat_session_t *s;
4633   clib_bihash_8_8_t *t;
4634
4635   if (sm->endpoint_dependent)
4636     return VNET_API_ERROR_UNSUPPORTED;
4637
4638   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4639   if (sm->num_workers > 1)
4640     tsm =
4641       vec_elt_at_index (sm->per_thread_data,
4642                         sm->worker_in2out_cb (&ip, fib_index, 0));
4643   else
4644     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4645
4646   init_nat_k (&kv, *addr, port, fib_index, proto);
4647   t = is_in ? &tsm->in2out : &tsm->out2in;
4648   if (!clib_bihash_search_8_8 (t, &kv, &value))
4649     {
4650       if (pool_is_free_index (tsm->sessions, value.value))
4651         return VNET_API_ERROR_UNSPECIFIED;
4652
4653       s = pool_elt_at_index (tsm->sessions, value.value);
4654       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4655       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4656       return 0;
4657     }
4658
4659   return VNET_API_ERROR_NO_SUCH_ENTRY;
4660 }
4661
4662 int
4663 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4664                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4665                       u32 vrf_id, int is_in)
4666 {
4667   ip4_header_t ip;
4668   clib_bihash_16_8_t *t;
4669   clib_bihash_kv_16_8_t kv, value;
4670   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4671   snat_session_t *s;
4672   snat_main_per_thread_data_t *tsm;
4673
4674   if (!sm->endpoint_dependent)
4675     return VNET_API_ERROR_FEATURE_DISABLED;
4676
4677   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4678   if (sm->num_workers > 1)
4679     tsm =
4680       vec_elt_at_index (sm->per_thread_data,
4681                         sm->worker_in2out_cb (&ip, fib_index, 0));
4682   else
4683     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4684
4685   t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
4686   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
4687   if (clib_bihash_search_16_8 (t, &kv, &value))
4688     {
4689       return VNET_API_ERROR_NO_SUCH_ENTRY;
4690     }
4691
4692   if (pool_is_free_index (tsm->sessions, value.value))
4693     return VNET_API_ERROR_UNSPECIFIED;
4694   s = pool_elt_at_index (tsm->sessions, value.value);
4695   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4696   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
4697   return 0;
4698 }
4699
4700 void
4701 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4702 {
4703   snat_main_t *sm = &snat_main;
4704
4705   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4706   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4707   sm->psid = psid;
4708   sm->psid_offset = psid_offset;
4709   sm->psid_length = psid_length;
4710 }
4711
4712 void
4713 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4714 {
4715   snat_main_t *sm = &snat_main;
4716
4717   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4718   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4719   sm->start_port = start_port;
4720   sm->end_port = end_port;
4721 }
4722
4723 void
4724 nat_set_alloc_addr_and_port_default (void)
4725 {
4726   snat_main_t *sm = &snat_main;
4727
4728   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4729   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4730 }
4731
4732 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4733                                  vlib_node_runtime_t * node,
4734                                  vlib_frame_t * frame)
4735 {
4736   return 0;
4737 }
4738
4739 /* *INDENT-OFF* */
4740 VLIB_REGISTER_NODE (nat_default_node) = {
4741   .name = "nat-default",
4742   .vector_size = sizeof (u32),
4743   .format_trace = 0,
4744   .type = VLIB_NODE_TYPE_INTERNAL,
4745   .n_errors = 0,
4746   .n_next_nodes = NAT_N_NEXT,
4747   .next_nodes = {
4748     [NAT_NEXT_DROP] = "error-drop",
4749     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4750     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4751     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4752     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
4753     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4754     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4755     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4756     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4757     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4758   },
4759 };
4760 /* *INDENT-ON* */
4761
4762 /*
4763  * fd.io coding-style-patch-verification: ON
4764  *
4765  * Local Variables:
4766  * eval: (c-set-style "gnu")
4767  * End:
4768  */