src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/lib/ipfix_logging.h>
  25 #include <nat/lib/nat_syslog.h>
  26 #include <nat/nat_inlines.h>
  27 #include <nat/nat44/inlines.h>
  28 #include <nat/nat_affinity.h>
  29 #include <nat/nat_ha.h>
  30 #include <vnet/fib/fib_table.h>
  31 #include <vnet/fib/ip4_fib.h>
  32 #include <vnet/ip/reass/ip4_sv_reass.h>
  33 #include <vppinfra/bihash_16_8.h>
  34 #include <nat/nat44/ed_inlines.h>
  35 #include <vnet/ip/ip_table.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 /* *INDENT-OFF* */
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat-pre-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  47                                "ip4-sv-reassembly-feature"),
  48 };
  49 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  50   .arc_name = "ip4-unicast",
  51   .node_name = "nat-pre-out2in",
  52   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  53                                "ip4-dhcp-client-detect",
  54                                "ip4-sv-reassembly-feature"),
  55 };
  56 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  57   .arc_name = "ip4-unicast",
  58   .node_name = "nat44-in2out-worker-handoff",
  59   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  60 };
  61 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  62   .arc_name = "ip4-unicast",
  63   .node_name = "nat44-out2in-worker-handoff",
  64   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  65                                "ip4-dhcp-client-detect"),
  66 };
  67 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  68   .arc_name = "ip4-unicast",
  69   .node_name = "nat44-in2out",
  70   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  71 };
  72 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  73   .arc_name = "ip4-unicast",
  74   .node_name = "nat44-out2in",
  75   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  76                                "ip4-dhcp-client-detect"),
  77 };
  78 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  79   .arc_name = "ip4-unicast",
  80   .node_name = "nat44-classify",
  81   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  82 };
  83 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-ed-in2out",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-ed-out2in",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  92                                "ip4-dhcp-client-detect"),
  93 };
  94 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-ed-classify",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  98 };
  99 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 100   .arc_name = "ip4-unicast",
 101   .node_name = "nat44-handoff-classify",
 102   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out-fast",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 108 };
 109 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in-fast",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-hairpin-dst",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-hairpin-dst",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 124 };
 125
 126 /* Hook up output features */
 127 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 128   .arc_name = "ip4-output",
 129   .node_name = "nat44-in2out-output",
 130   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 133   .arc_name = "ip4-output",
 134   .node_name = "nat44-in2out-output-worker-handoff",
 135   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 136 };
 137 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 138   .arc_name = "ip4-output",
 139   .node_name = "nat44-hairpin-src",
 140   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 141 };
 142 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 143   .arc_name = "ip4-output",
 144   .node_name = "nat-pre-in2out-output",
 145   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 146   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 147 };
 148 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-ed-in2out-output",
 151   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 152   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 153 };
 154 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 155   .arc_name = "ip4-output",
 156   .node_name = "nat44-ed-hairpin-src",
 157   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 158   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 159 };
 160
 161 /* Hook up ip4-local features */
 162 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 163 {
 164   .arc_name = "ip4-local",
 165   .node_name = "nat44-hairpinning",
 166   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 167 };
 168 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 169 {
 170   .arc_name = "ip4-local",
 171   .node_name = "nat44-ed-hairpinning",
 172   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 173 };
 174
 175
 176 VLIB_PLUGIN_REGISTER () = {
 177     .version = VPP_BUILD_VER,
 178     .description = "Network Address Translation (NAT)",
 179 };
 180 /* *INDENT-ON* */
 181
 182 static u32
 183 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
 184                                u32 rx_fib_index, u8 is_output);
 185
 186 static u32
 187 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
 188                                u8 is_output);
 189
 190 static u32
 191 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
 192                            u32 rx_fib_index0, u8 is_output);
 193
 194 static u32
 195 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
 196                            u8 is_output);
 197
 198 static u32 nat_calc_bihash_buckets (u32 n_elts);
 199
 200 u8 *
 201 format_session_kvp (u8 * s, va_list * args)
 202 {
 203   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 204
 205   s = format (s, "%U session-index %llu", format_snat_key, v->key, v->value);
 206
 207   return s;
 208 }
 209
 210 u8 *
 211 format_static_mapping_kvp (u8 * s, va_list * args)
 212 {
 213   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 214
 215   s = format (s, "%U static-mapping-index %llu",
 216               format_snat_key, v->key, v->value);
 217
 218   return s;
 219 }
 220
 221 u8 *
 222 format_user_kvp (u8 * s, va_list * args)
 223 {
 224   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 225   snat_user_key_t k;
 226
 227   k.as_u64 = v->key;
 228
 229   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
 230               k.fib_index, v->value);
 231
 232   return s;
 233 }
 234
 235 u8 *
 236 format_ed_session_kvp (u8 * s, va_list * args)
 237 {
 238   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 239
 240   u8 proto;
 241   u16 r_port, l_port;
 242   ip4_address_t l_addr, r_addr;
 243   u32 fib_index;
 244
 245   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 246   s =
 247     format (s,
 248             "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
 249             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 250             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 251             format_ip_protocol, proto, fib_index,
 252             ed_value_get_session_index (v), ed_value_get_thread_index (v));
 253
 254   return s;
 255 }
 256
 257 void
 258 nat44_ei_free_session_data (snat_main_t * sm, snat_session_t * s,
 259                             u32 thread_index, u8 is_ha)
 260 {
 261   clib_bihash_kv_8_8_t kv;
 262
 263   snat_main_per_thread_data_t *tsm =
 264     vec_elt_at_index (sm->per_thread_data, thread_index);
 265
 266   init_nat_i2o_k (&kv, s);
 267   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 268     nat_elog_warn ("in2out key del failed");
 269
 270   init_nat_o2i_k (&kv, s);
 271   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 272     nat_elog_warn ("out2in key del failed");
 273
 274   if (!is_ha)
 275     {
 276       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 277                                &s->in2out.addr, s->in2out.port,
 278                                &s->out2in.addr, s->out2in.port, s->nat_proto);
 279
 280       nat_ipfix_logging_nat44_ses_delete (thread_index,
 281                                           s->in2out.addr.as_u32,
 282                                           s->out2in.addr.as_u32,
 283                                           s->nat_proto,
 284                                           s->in2out.port,
 285                                           s->out2in.port,
 286                                           s->in2out.fib_index);
 287
 288       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 289                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 290                    thread_index);
 291
 292     }
 293
 294   if (snat_is_session_static (s))
 295     return;
 296
 297   snat_free_outside_address_and_port (sm->addresses, thread_index,
 298                                       &s->out2in.addr, s->out2in.port,
 299                                       s->nat_proto);
 300 }
 301
 302 static_always_inline void
 303 nat44_ei_user_del_sessions (snat_user_t * u, u32 thread_index)
 304 {
 305   dlist_elt_t *elt;
 306   snat_session_t *s;
 307
 308   snat_main_t *sm = &snat_main;
 309   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 310
 311   // get head
 312   elt = pool_elt_at_index (tsm->list_pool,
 313                            u->sessions_per_user_list_head_index);
 314   // get first element
 315   elt = pool_elt_at_index (tsm->list_pool, elt->next);
 316
 317   while (elt->value != ~0)
 318     {
 319       s = pool_elt_at_index (tsm->sessions, elt->value);
 320       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 321
 322       nat44_ei_free_session_data (sm, s, thread_index, 0);
 323       nat44_delete_session (sm, s, thread_index);
 324     }
 325 }
 326
 327 int
 328 nat44_ei_user_del (ip4_address_t * addr, u32 fib_index)
 329 {
 330   int rv = 1;
 331
 332   snat_main_t *sm = &snat_main;
 333   snat_main_per_thread_data_t *tsm;
 334
 335   snat_user_key_t user_key;
 336   clib_bihash_kv_8_8_t kv, value;
 337
 338   if (sm->endpoint_dependent)
 339     return rv;
 340
 341   user_key.addr.as_u32 = addr->as_u32;
 342   user_key.fib_index = fib_index;
 343   kv.key = user_key.as_u64;
 344
 345   if (sm->num_workers > 1)
 346     {
 347       /* *INDENT-OFF* */
 348       vec_foreach (tsm, sm->per_thread_data)
 349         {
 350           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 351             {
 352               nat44_ei_user_del_sessions (
 353                   pool_elt_at_index (tsm->users, value.value),
 354                   tsm->thread_index);
 355               rv = 0;
 356               break;
 357             }
 358         }
 359       /* *INDENT-ON* */
 360     }
 361   else
 362     {
 363       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 364       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 365         {
 366           nat44_ei_user_del_sessions (pool_elt_at_index
 367                                       (tsm->users, value.value),
 368                                       tsm->thread_index);
 369           rv = 0;
 370         }
 371     }
 372   return rv;
 373 }
 374
 375 void
 376 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 377                        u8 is_ha)
 378 {
 379   clib_bihash_kv_8_8_t kv;
 380   u8 proto;
 381   u16 r_port, l_port;
 382   ip4_address_t *l_addr, *r_addr;
 383   u32 fib_index = 0;
 384   clib_bihash_kv_16_8_t ed_kv;
 385   snat_main_per_thread_data_t *tsm =
 386     vec_elt_at_index (sm->per_thread_data, thread_index);
 387
 388   if (is_ed_session (s))
 389     {
 390       per_vrf_sessions_unregister_session (s, thread_index);
 391     }
 392
 393   if (is_fwd_bypass_session (s))
 394     {
 395       if (snat_is_unk_proto_session (s))
 396         {
 397           init_ed_k (&ed_kv, s->in2out.addr, 0, s->ext_host_addr, 0, 0,
 398                      s->in2out.port);
 399         }
 400       else
 401         {
 402           l_port = s->in2out.port;
 403           r_port = s->ext_host_port;
 404           l_addr = &s->in2out.addr;
 405           r_addr = &s->ext_host_addr;
 406           proto = nat_proto_to_ip_proto (s->nat_proto);
 407           fib_index = s->in2out.fib_index;
 408           init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index,
 409                      proto);
 410         }
 411       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 412         nat_elog_warn ("in2out_ed key del failed");
 413       return;
 414     }
 415
 416   /* session lookup tables */
 417   if (is_ed_session (s))
 418     {
 419       if (is_affinity_sessions (s))
 420         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 421                              s->nat_proto, s->out2in.port);
 422       l_addr = &s->out2in.addr;
 423       r_addr = &s->ext_host_addr;
 424       fib_index = s->out2in.fib_index;
 425       if (snat_is_unk_proto_session (s))
 426         {
 427           proto = s->in2out.port;
 428           r_port = 0;
 429           l_port = 0;
 430         }
 431       else
 432         {
 433           proto = nat_proto_to_ip_proto (s->nat_proto);
 434           l_port = s->out2in.port;
 435           r_port = s->ext_host_port;
 436         }
 437       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 438       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 439         nat_elog_warn ("out2in_ed key del failed");
 440       l_addr = &s->in2out.addr;
 441       fib_index = s->in2out.fib_index;
 442       if (!snat_is_unk_proto_session (s))
 443         l_port = s->in2out.port;
 444       if (is_twice_nat_session (s))
 445         {
 446           r_addr = &s->ext_host_nat_addr;
 447           r_port = s->ext_host_nat_port;
 448         }
 449       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 450       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 451         nat_elog_warn ("in2out_ed key del failed");
 452
 453       if (!is_ha)
 454         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 455                                &s->in2out.addr, s->in2out.port,
 456                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 457                                &s->out2in.addr, s->out2in.port,
 458                                &s->ext_host_addr, s->ext_host_port,
 459                                s->nat_proto, is_twice_nat_session (s));
 460     }
 461   else
 462     {
 463       init_nat_i2o_k (&kv, s);
 464       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 465         nat_elog_warn ("in2out key del failed");
 466       init_nat_o2i_k (&kv, s);
 467       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 468         nat_elog_warn ("out2in key del failed");
 469
 470       if (!is_ha)
 471         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 472                                  &s->in2out.addr, s->in2out.port,
 473                                  &s->out2in.addr, s->out2in.port,
 474                                  s->nat_proto);
 475     }
 476
 477   if (snat_is_unk_proto_session (s))
 478     return;
 479
 480   if (!is_ha)
 481     {
 482       /* log NAT event */
 483       nat_ipfix_logging_nat44_ses_delete (thread_index,
 484                                           s->in2out.addr.as_u32,
 485                                           s->out2in.addr.as_u32,
 486                                           s->nat_proto,
 487                                           s->in2out.port,
 488                                           s->out2in.port,
 489                                           s->in2out.fib_index);
 490
 491       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 492                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 493                    thread_index);
 494     }
 495
 496   /* Twice NAT address and port for external host */
 497   if (is_twice_nat_session (s))
 498     {
 499       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 500                                           thread_index,
 501                                           &s->ext_host_nat_addr,
 502                                           s->ext_host_nat_port, s->nat_proto);
 503     }
 504
 505   if (snat_is_session_static (s))
 506     return;
 507
 508   snat_free_outside_address_and_port (sm->addresses, thread_index,
 509                                       &s->out2in.addr, s->out2in.port,
 510                                       s->nat_proto);
 511 }
 512
 513 snat_user_t *
 514 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 515                         u32 thread_index)
 516 {
 517   snat_user_t *u = 0;
 518   snat_user_key_t user_key;
 519   clib_bihash_kv_8_8_t kv, value;
 520   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 521   dlist_elt_t *per_user_list_head_elt;
 522
 523   user_key.addr.as_u32 = addr->as_u32;
 524   user_key.fib_index = fib_index;
 525   kv.key = user_key.as_u64;
 526
 527   /* Ever heard of the "user" = src ip4 address before? */
 528   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 529     {
 530       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 531         {
 532           vlib_increment_simple_counter (&sm->user_limit_reached,
 533                                          thread_index, 0, 1);
 534           nat_elog_warn ("maximum user limit reached");
 535           return NULL;
 536         }
 537       /* no, make a new one */
 538       pool_get (tsm->users, u);
 539       clib_memset (u, 0, sizeof (*u));
 540
 541       u->addr.as_u32 = addr->as_u32;
 542       u->fib_index = fib_index;
 543
 544       pool_get (tsm->list_pool, per_user_list_head_elt);
 545
 546       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 547         tsm->list_pool;
 548
 549       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 550
 551       kv.value = u - tsm->users;
 552
 553       /* add user */
 554       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 555         {
 556           nat_elog_warn ("user_hash key add failed");
 557           nat44_delete_user_with_no_session (sm, u, thread_index);
 558           return NULL;
 559         }
 560
 561       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 562                                pool_elts (tsm->users));
 563     }
 564   else
 565     {
 566       u = pool_elt_at_index (tsm->users, value.value);
 567     }
 568
 569   return u;
 570 }
 571
 572 // only NAT EI
 573 snat_session_t *
 574 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 575                               u32 thread_index, f64 now)
 576 {
 577   snat_session_t *s;
 578   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 579   u32 oldest_per_user_translation_list_index, session_index;
 580   dlist_elt_t *oldest_per_user_translation_list_elt;
 581   dlist_elt_t *per_user_translation_list_elt;
 582
 583   /* Over quota? Recycle the least recently used translation */
 584   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 585     {
 586       oldest_per_user_translation_list_index =
 587         clib_dlist_remove_head (tsm->list_pool,
 588                                 u->sessions_per_user_list_head_index);
 589
 590       ASSERT (oldest_per_user_translation_list_index != ~0);
 591
 592       /* Add it back to the end of the LRU list */
 593       clib_dlist_addtail (tsm->list_pool,
 594                           u->sessions_per_user_list_head_index,
 595                           oldest_per_user_translation_list_index);
 596       /* Get the list element */
 597       oldest_per_user_translation_list_elt =
 598         pool_elt_at_index (tsm->list_pool,
 599                            oldest_per_user_translation_list_index);
 600
 601       /* Get the session index from the list element */
 602       session_index = oldest_per_user_translation_list_elt->value;
 603
 604       /* Get the session */
 605       s = pool_elt_at_index (tsm->sessions, session_index);
 606
 607       // TODO: ONLY EI version should be called
 608       nat_free_session_data (sm, s, thread_index, 0);
 609       if (snat_is_session_static (s))
 610         u->nstaticsessions--;
 611       else
 612         u->nsessions--;
 613       s->flags = 0;
 614       s->total_bytes = 0;
 615       s->total_pkts = 0;
 616       s->state = 0;
 617       s->ext_host_addr.as_u32 = 0;
 618       s->ext_host_port = 0;
 619       s->ext_host_nat_addr.as_u32 = 0;
 620       s->ext_host_nat_port = 0;
 621     }
 622   else
 623     {
 624       pool_get (tsm->sessions, s);
 625       clib_memset (s, 0, sizeof (*s));
 626
 627       /* Create list elts */
 628       pool_get (tsm->list_pool, per_user_translation_list_elt);
 629       clib_dlist_init (tsm->list_pool,
 630                        per_user_translation_list_elt - tsm->list_pool);
 631
 632       per_user_translation_list_elt->value = s - tsm->sessions;
 633       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 634       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 635
 636       clib_dlist_addtail (tsm->list_pool,
 637                           s->per_user_list_head_index,
 638                           per_user_translation_list_elt - tsm->list_pool);
 639
 640       s->user_index = u - tsm->users;
 641       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 642                                pool_elts (tsm->sessions));
 643     }
 644
 645   s->ha_last_refreshed = now;
 646
 647   return s;
 648 }
 649
 650 void
 651 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 652                           int is_add)
 653 {
 654   snat_main_t *sm = &snat_main;
 655   fib_prefix_t prefix = {
 656     .fp_len = p_len,
 657     .fp_proto = FIB_PROTOCOL_IP4,
 658     .fp_addr = {
 659                 .ip4.as_u32 = addr->as_u32,
 660                 },
 661   };
 662   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 663
 664   if (is_add)
 665     fib_table_entry_update_one_path (fib_index,
 666                                      &prefix,
 667                                      sm->fib_src_low,
 668                                      (FIB_ENTRY_FLAG_CONNECTED |
 669                                       FIB_ENTRY_FLAG_LOCAL |
 670                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 671                                      DPO_PROTO_IP4,
 672                                      NULL,
 673                                      sw_if_index,
 674                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 675   else
 676     fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 677 }
 678
 679 int
 680 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 681                   u8 twice_nat)
 682 {
 683   snat_address_t *ap;
 684   snat_interface_t *i;
 685   vlib_thread_main_t *tm = vlib_get_thread_main ();
 686
 687   if (twice_nat && !sm->endpoint_dependent)
 688     {
 689       nat_log_err ("unsupported");
 690       return VNET_API_ERROR_UNSUPPORTED;
 691     }
 692
 693   /* Check if address already exists */
 694   /* *INDENT-OFF* */
 695   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 696     {
 697       if (ap->addr.as_u32 == addr->as_u32)
 698         {
 699           nat_log_err ("address exist");
 700           return VNET_API_ERROR_VALUE_EXIST;
 701         }
 702     }
 703   /* *INDENT-ON* */
 704
 705   if (twice_nat)
 706     vec_add2 (sm->twice_nat_addresses, ap, 1);
 707   else
 708     vec_add2 (sm->addresses, ap, 1);
 709
 710   ap->addr = *addr;
 711   if (vrf_id != ~0)
 712     ap->fib_index =
 713       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 714                                          sm->fib_src_low);
 715   else
 716     ap->fib_index = ~0;
 717
 718   /* *INDENT-OFF* */
 719   #define _(N, i, n, s) \
 720     clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 721     ap->busy_##n##_ports = 0; \
 722     ap->busy_##n##_ports_per_thread = 0;\
 723     vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 724     foreach_nat_protocol
 725   #undef _
 726   /* *INDENT-ON* */
 727
 728   if (twice_nat)
 729     return 0;
 730
 731   /* Add external address to FIB */
 732   /* *INDENT-OFF* */
 733   pool_foreach (i, sm->interfaces,
 734   ({
 735     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 736       continue;
 737
 738     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 739     break;
 740   }));
 741   pool_foreach (i, sm->output_feature_interfaces,
 742   ({
 743     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 744       continue;
 745
 746     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 747     break;
 748   }));
 749   /* *INDENT-ON* */
 750
 751   return 0;
 752 }
 753
 754 static int
 755 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 756 {
 757   snat_static_mapping_t *m;
 758   /* *INDENT-OFF* */
 759   pool_foreach (m, sm->static_mappings,
 760   ({
 761       if (is_addr_only_static_mapping (m) ||
 762           is_out2in_only_static_mapping (m) ||
 763           is_identity_static_mapping (m))
 764         continue;
 765       if (m->external_addr.as_u32 == addr.as_u32)
 766         return 1;
 767   }));
 768   /* *INDENT-ON* */
 769
 770   return 0;
 771 }
 772
 773 static void
 774 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 775                                        ip4_address_t l_addr,
 776                                        u16 l_port,
 777                                        u32 sw_if_index,
 778                                        u16 e_port,
 779                                        u32 vrf_id,
 780                                        nat_protocol_t proto,
 781                                        int addr_only, int is_add, u8 * tag,
 782                                        int twice_nat, int out2in_only,
 783                                        int identity_nat,
 784                                        ip4_address_t pool_addr, int exact)
 785 {
 786   snat_static_map_resolve_t *rp;
 787
 788   vec_add2 (sm->to_resolve, rp, 1);
 789   rp->l_addr.as_u32 = l_addr.as_u32;
 790   rp->l_port = l_port;
 791   rp->sw_if_index = sw_if_index;
 792   rp->e_port = e_port;
 793   rp->vrf_id = vrf_id;
 794   rp->proto = proto;
 795   rp->addr_only = addr_only;
 796   rp->is_add = is_add;
 797   rp->twice_nat = twice_nat;
 798   rp->out2in_only = out2in_only;
 799   rp->identity_nat = identity_nat;
 800   rp->tag = vec_dup (tag);
 801   rp->pool_addr = pool_addr;
 802   rp->exact = exact;
 803 }
 804
 805 static u32
 806 get_thread_idx_by_port (u16 e_port)
 807 {
 808   snat_main_t *sm = &snat_main;
 809   u32 thread_idx = sm->num_workers;
 810   if (sm->num_workers > 1)
 811     {
 812       thread_idx =
 813         sm->first_worker_index +
 814         sm->workers[(e_port - 1024) / sm->port_per_thread];
 815     }
 816   return thread_idx;
 817 }
 818
 819 void
 820 nat_ei_static_mapping_del_sessions (snat_main_t * sm,
 821                                     snat_main_per_thread_data_t * tsm,
 822                                     snat_user_key_t u_key, int addr_only,
 823                                     ip4_address_t e_addr, u16 e_port)
 824 {
 825   clib_bihash_kv_8_8_t kv, value;
 826   kv.key = u_key.as_u64;
 827   u64 user_index;
 828   dlist_elt_t *head, *elt;
 829   snat_user_t *u;
 830   snat_session_t *s;
 831   u32 elt_index, head_index, ses_index;
 832
 833   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 834     {
 835       user_index = value.value;
 836       u = pool_elt_at_index (tsm->users, user_index);
 837       if (u->nstaticsessions)
 838         {
 839           head_index = u->sessions_per_user_list_head_index;
 840           head = pool_elt_at_index (tsm->list_pool, head_index);
 841           elt_index = head->next;
 842           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 843           ses_index = elt->value;
 844           while (ses_index != ~0)
 845             {
 846               s = pool_elt_at_index (tsm->sessions, ses_index);
 847               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 848               ses_index = elt->value;
 849
 850               if (!addr_only)
 851                 {
 852                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 853                       (s->out2in.port != e_port))
 854                     continue;
 855                 }
 856
 857               if (is_lb_session (s))
 858                 continue;
 859
 860               if (!snat_is_session_static (s))
 861                 continue;
 862
 863               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 864               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 865
 866               if (!addr_only)
 867                 break;
 868             }
 869         }
 870     }
 871 }
 872
 873 void
 874 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 875                                     snat_main_per_thread_data_t * tsm,
 876                                     ip4_address_t l_addr,
 877                                     u16 l_port,
 878                                     u8 protocol,
 879                                     u32 fib_index, int addr_only,
 880                                     ip4_address_t e_addr, u16 e_port)
 881 {
 882   snat_session_t *s;
 883   u32 *indexes_to_free = NULL;
 884   /* *INDENT-OFF* */
 885   pool_foreach (s, tsm->sessions, {
 886     if (s->in2out.fib_index != fib_index ||
 887         s->in2out.addr.as_u32 != l_addr.as_u32)
 888       {
 889         continue;
 890       }
 891     if (!addr_only)
 892       {
 893         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 894             s->out2in.port != e_port ||
 895             s->in2out.port != l_port ||
 896             s->nat_proto != protocol)
 897           continue;
 898       }
 899
 900     if (is_lb_session (s))
 901       continue;
 902     if (!snat_is_session_static (s))
 903       continue;
 904     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 905     vec_add1 (indexes_to_free, s - tsm->sessions);
 906     if (!addr_only)
 907       break;
 908   });
 909   /* *INDENT-ON* */
 910   u32 *ses_index;
 911   vec_foreach (ses_index, indexes_to_free)
 912   {
 913     s = pool_elt_at_index (tsm->sessions, *ses_index);
 914     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 915   }
 916   vec_free (indexes_to_free);
 917 }
 918
 919 int
 920 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 921                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 922                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 923                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 924                          u8 identity_nat, ip4_address_t pool_addr, int exact)
 925 {
 926   snat_main_t *sm = &snat_main;
 927   snat_static_mapping_t *m;
 928   clib_bihash_kv_8_8_t kv, value;
 929   snat_address_t *a = 0;
 930   u32 fib_index = ~0;
 931   snat_interface_t *interface;
 932   int i;
 933   snat_main_per_thread_data_t *tsm;
 934   snat_user_key_t u_key;
 935   snat_user_t *u;
 936   dlist_elt_t *head, *elt;
 937   u32 elt_index, head_index;
 938   u32 ses_index;
 939   u64 user_index;
 940   snat_session_t *s;
 941   snat_static_map_resolve_t *rp, *rp_match = 0;
 942   nat44_lb_addr_port_t *local;
 943   u32 find = ~0;
 944
 945   if (!sm->endpoint_dependent)
 946     {
 947       if (twice_nat || out2in_only)
 948         return VNET_API_ERROR_UNSUPPORTED;
 949     }
 950
 951   /* If the external address is a specific interface address */
 952   if (sw_if_index != ~0)
 953     {
 954       ip4_address_t *first_int_addr;
 955
 956       for (i = 0; i < vec_len (sm->to_resolve); i++)
 957         {
 958           rp = sm->to_resolve + i;
 959           if (rp->sw_if_index != sw_if_index ||
 960               rp->l_addr.as_u32 != l_addr.as_u32 ||
 961               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 962             continue;
 963
 964           if (!addr_only)
 965             {
 966               if ((rp->l_port != l_port && rp->e_port != e_port)
 967                   || rp->proto != proto)
 968                 continue;
 969             }
 970
 971           rp_match = rp;
 972           break;
 973         }
 974
 975       /* Might be already set... */
 976       first_int_addr = ip4_interface_first_address
 977         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 978
 979       if (is_add)
 980         {
 981           if (rp_match)
 982             return VNET_API_ERROR_VALUE_EXIST;
 983
 984           snat_add_static_mapping_when_resolved
 985             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 986              addr_only, is_add, tag, twice_nat, out2in_only,
 987              identity_nat, pool_addr, exact);
 988
 989           /* DHCP resolution required? */
 990           if (first_int_addr == 0)
 991             {
 992               return 0;
 993             }
 994           else
 995             {
 996               e_addr.as_u32 = first_int_addr->as_u32;
 997               /* Identity mapping? */
 998               if (l_addr.as_u32 == 0)
 999                 l_addr.as_u32 = e_addr.as_u32;
1000             }
1001         }
1002       else
1003         {
1004           if (!rp_match)
1005             return VNET_API_ERROR_NO_SUCH_ENTRY;
1006
1007           vec_del1 (sm->to_resolve, i);
1008
1009           if (first_int_addr)
1010             {
1011               e_addr.as_u32 = first_int_addr->as_u32;
1012               /* Identity mapping? */
1013               if (l_addr.as_u32 == 0)
1014                 l_addr.as_u32 = e_addr.as_u32;
1015             }
1016           else
1017             return 0;
1018         }
1019     }
1020
1021   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
1022   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1023     m = 0;
1024   else
1025     m = pool_elt_at_index (sm->static_mappings, value.value);
1026
1027   if (is_add)
1028     {
1029       if (m)
1030         {
1031           if (is_identity_static_mapping (m))
1032             {
1033               /* *INDENT-OFF* */
1034               pool_foreach (local, m->locals,
1035               ({
1036                 if (local->vrf_id == vrf_id)
1037                   return VNET_API_ERROR_VALUE_EXIST;
1038               }));
1039               /* *INDENT-ON* */
1040               pool_get (m->locals, local);
1041               local->vrf_id = vrf_id;
1042               local->fib_index =
1043                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1044                                                    sm->fib_src_low);
1045               init_nat_kv (&kv, m->local_addr, m->local_port,
1046                            local->fib_index, m->proto,
1047                            m - sm->static_mappings);
1048               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1049               return 0;
1050             }
1051           else
1052             return VNET_API_ERROR_VALUE_EXIST;
1053         }
1054
1055       if (twice_nat && addr_only)
1056         return VNET_API_ERROR_UNSUPPORTED;
1057
1058       /* Convert VRF id to FIB index */
1059       if (vrf_id != ~0)
1060         fib_index =
1061           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1062                                              sm->fib_src_low);
1063       /* If not specified use inside VRF id from SNAT plugin startup config */
1064       else
1065         {
1066           fib_index = sm->inside_fib_index;
1067           vrf_id = sm->inside_vrf_id;
1068           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1069         }
1070
1071       if (!(out2in_only || identity_nat))
1072         {
1073           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
1074                       addr_only ? 0 : proto);
1075           if (!clib_bihash_search_8_8
1076               (&sm->static_mapping_by_local, &kv, &value))
1077             return VNET_API_ERROR_VALUE_EXIST;
1078         }
1079
1080       /* Find external address in allocated addresses and reserve port for
1081          address and port pair mapping when dynamic translations enabled */
1082       if (!(addr_only || sm->static_mapping_only || out2in_only))
1083         {
1084           for (i = 0; i < vec_len (sm->addresses); i++)
1085             {
1086               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1087                 {
1088                   a = sm->addresses + i;
1089                   /* External port must be unused */
1090                   switch (proto)
1091                     {
1092 #define _(N, j, n, s) \
1093                     case NAT_PROTOCOL_##N: \
1094                       if (a->busy_##n##_port_refcounts[e_port]) \
1095                         return VNET_API_ERROR_INVALID_VALUE; \
1096                       ++a->busy_##n##_port_refcounts[e_port]; \
1097                       if (e_port > 1024) \
1098                         { \
1099                           a->busy_##n##_ports++; \
1100                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1101                         } \
1102                       break;
1103                       foreach_nat_protocol
1104 #undef _
1105                     default:
1106                       nat_elog_info ("unknown protocol");
1107                       return VNET_API_ERROR_INVALID_VALUE_2;
1108                     }
1109                   break;
1110                 }
1111             }
1112           /* External address must be allocated */
1113           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1114             {
1115               if (sw_if_index != ~0)
1116                 {
1117                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1118                     {
1119                       rp = sm->to_resolve + i;
1120                       if (rp->addr_only)
1121                         continue;
1122                       if (rp->sw_if_index != sw_if_index &&
1123                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1124                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1125                           rp->e_port != e_port && rp->proto != proto)
1126                         continue;
1127
1128                       vec_del1 (sm->to_resolve, i);
1129                       break;
1130                     }
1131                 }
1132               return VNET_API_ERROR_NO_SUCH_ENTRY;
1133             }
1134         }
1135
1136       pool_get (sm->static_mappings, m);
1137       clib_memset (m, 0, sizeof (*m));
1138       m->tag = vec_dup (tag);
1139       m->local_addr = l_addr;
1140       m->external_addr = e_addr;
1141       m->twice_nat = twice_nat;
1142
1143       if (twice_nat == TWICE_NAT && exact)
1144         {
1145           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
1146           m->pool_addr = pool_addr;
1147         }
1148
1149       if (out2in_only)
1150         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1151       if (addr_only)
1152         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1153       if (identity_nat)
1154         {
1155           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1156           pool_get (m->locals, local);
1157           local->vrf_id = vrf_id;
1158           local->fib_index = fib_index;
1159         }
1160       else
1161         {
1162           m->vrf_id = vrf_id;
1163           m->fib_index = fib_index;
1164         }
1165       if (!addr_only)
1166         {
1167           m->local_port = l_port;
1168           m->external_port = e_port;
1169           m->proto = proto;
1170         }
1171
1172       if (sm->num_workers > 1)
1173         {
1174           ip4_header_t ip = {
1175             .src_address = m->local_addr,
1176           };
1177           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1178           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1179         }
1180       else
1181         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1182
1183       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
1184                    m - sm->static_mappings);
1185       if (!out2in_only)
1186         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1187
1188       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1189                    m - sm->static_mappings);
1190       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1191
1192       /* Delete dynamic sessions matching local address (+ local port) */
1193       // TODO: based on type of NAT EI/ED
1194       if (!(sm->static_mapping_only))
1195         {
1196           u_key.addr = m->local_addr;
1197           u_key.fib_index = m->fib_index;
1198           kv.key = u_key.as_u64;
1199           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1200             {
1201               user_index = value.value;
1202               u = pool_elt_at_index (tsm->users, user_index);
1203               if (u->nsessions)
1204                 {
1205                   head_index = u->sessions_per_user_list_head_index;
1206                   head = pool_elt_at_index (tsm->list_pool, head_index);
1207                   elt_index = head->next;
1208                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1209                   ses_index = elt->value;
1210                   while (ses_index != ~0)
1211                     {
1212                       s = pool_elt_at_index (tsm->sessions, ses_index);
1213                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1214                       ses_index = elt->value;
1215
1216                       if (snat_is_session_static (s))
1217                         continue;
1218
1219                       if (!addr_only && s->in2out.port != m->local_port)
1220                         continue;
1221
1222                       nat_free_session_data (sm, s,
1223                                              tsm - sm->per_thread_data, 0);
1224                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1225
1226                       if (!addr_only && !sm->endpoint_dependent)
1227                         break;
1228                     }
1229                 }
1230             }
1231         }
1232     }
1233   else
1234     {
1235       if (!m)
1236         {
1237           if (sw_if_index != ~0)
1238             return 0;
1239           else
1240             return VNET_API_ERROR_NO_SUCH_ENTRY;
1241         }
1242
1243       if (identity_nat)
1244         {
1245           if (vrf_id == ~0)
1246             vrf_id = sm->inside_vrf_id;
1247
1248           /* *INDENT-OFF* */
1249           pool_foreach (local, m->locals,
1250           ({
1251             if (local->vrf_id == vrf_id)
1252               find = local - m->locals;
1253           }));
1254           /* *INDENT-ON* */
1255           if (find == ~0)
1256             return VNET_API_ERROR_NO_SUCH_ENTRY;
1257
1258           local = pool_elt_at_index (m->locals, find);
1259           fib_index = local->fib_index;
1260           pool_put (m->locals, local);
1261         }
1262       else
1263         fib_index = m->fib_index;
1264
1265       /* Free external address port */
1266       if (!(addr_only || sm->static_mapping_only || out2in_only))
1267         {
1268           for (i = 0; i < vec_len (sm->addresses); i++)
1269             {
1270               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1271                 {
1272                   a = sm->addresses + i;
1273                   switch (proto)
1274                     {
1275 #define _(N, j, n, s) \
1276                     case NAT_PROTOCOL_##N: \
1277                       --a->busy_##n##_port_refcounts[e_port]; \
1278                       if (e_port > 1024) \
1279                         { \
1280                           a->busy_##n##_ports--; \
1281                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1282                         } \
1283                       break;
1284                       foreach_nat_protocol
1285 #undef _
1286                     default:
1287                       nat_elog_info ("unknown protocol");
1288                       return VNET_API_ERROR_INVALID_VALUE_2;
1289                     }
1290                   break;
1291                 }
1292             }
1293         }
1294
1295       if (sm->num_workers > 1)
1296         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1297       else
1298         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1299
1300       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1301       if (!out2in_only)
1302         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1303
1304       /* Delete session(s) for static mapping if exist */
1305       if (!(sm->static_mapping_only) ||
1306           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1307         {
1308           if (sm->endpoint_dependent)
1309             {
1310               nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1311                                                   m->local_port, m->proto,
1312                                                   fib_index, addr_only,
1313                                                   e_addr, e_port);
1314             }
1315           else
1316             {
1317               u_key.addr = m->local_addr;
1318               u_key.fib_index = fib_index;
1319               kv.key = u_key.as_u64;
1320               nat_ei_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1321                                                   e_addr, e_port);
1322             }
1323         }
1324
1325       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1326       if (pool_elts (m->locals))
1327         return 0;
1328
1329       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1330       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1331
1332       vec_free (m->tag);
1333       vec_free (m->workers);
1334       /* Delete static mapping from pool */
1335       pool_put (sm->static_mappings, m);
1336     }
1337
1338   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1339     return 0;
1340
1341   /* Add/delete external address to FIB */
1342   /* *INDENT-OFF* */
1343   pool_foreach (interface, sm->interfaces,
1344   ({
1345     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1346       continue;
1347
1348     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1349     break;
1350   }));
1351   pool_foreach (interface, sm->output_feature_interfaces,
1352   ({
1353     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1354       continue;
1355
1356     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1357     break;
1358   }));
1359   /* *INDENT-ON* */
1360
1361   return 0;
1362 }
1363
1364 int
1365 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1366                                  nat_protocol_t proto,
1367                                  nat44_lb_addr_port_t * locals, u8 is_add,
1368                                  twice_nat_type_t twice_nat, u8 out2in_only,
1369                                  u8 * tag, u32 affinity)
1370 {
1371   snat_main_t *sm = &snat_main;
1372   snat_static_mapping_t *m;
1373   clib_bihash_kv_8_8_t kv, value;
1374   snat_address_t *a = 0;
1375   int i;
1376   nat44_lb_addr_port_t *local;
1377   snat_main_per_thread_data_t *tsm;
1378   snat_session_t *s;
1379   uword *bitmap = 0;
1380
1381   if (!sm->endpoint_dependent)
1382     return VNET_API_ERROR_UNSUPPORTED;
1383
1384   init_nat_k (&kv, e_addr, e_port, 0, proto);
1385   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1386     m = 0;
1387   else
1388     m = pool_elt_at_index (sm->static_mappings, value.value);
1389
1390   if (is_add)
1391     {
1392       if (m)
1393         return VNET_API_ERROR_VALUE_EXIST;
1394
1395       if (vec_len (locals) < 2)
1396         return VNET_API_ERROR_INVALID_VALUE;
1397
1398       /* Find external address in allocated addresses and reserve port for
1399          address and port pair mapping when dynamic translations enabled */
1400       if (!(sm->static_mapping_only || out2in_only))
1401         {
1402           for (i = 0; i < vec_len (sm->addresses); i++)
1403             {
1404               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1405                 {
1406                   a = sm->addresses + i;
1407                   /* External port must be unused */
1408                   switch (proto)
1409                     {
1410 #define _(N, j, n, s) \
1411                     case NAT_PROTOCOL_##N: \
1412                       if (a->busy_##n##_port_refcounts[e_port]) \
1413                         return VNET_API_ERROR_INVALID_VALUE; \
1414                       ++a->busy_##n##_port_refcounts[e_port]; \
1415                       if (e_port > 1024) \
1416                         { \
1417                           a->busy_##n##_ports++; \
1418                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1419                         } \
1420                       break;
1421                       foreach_nat_protocol
1422 #undef _
1423                     default:
1424                       nat_elog_info ("unknown protocol");
1425                       return VNET_API_ERROR_INVALID_VALUE_2;
1426                     }
1427                   break;
1428                 }
1429             }
1430           /* External address must be allocated */
1431           if (!a)
1432             return VNET_API_ERROR_NO_SUCH_ENTRY;
1433         }
1434
1435       pool_get (sm->static_mappings, m);
1436       clib_memset (m, 0, sizeof (*m));
1437       m->tag = vec_dup (tag);
1438       m->external_addr = e_addr;
1439       m->external_port = e_port;
1440       m->proto = proto;
1441       m->twice_nat = twice_nat;
1442       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1443       if (out2in_only)
1444         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1445       m->affinity = affinity;
1446
1447       if (affinity)
1448         m->affinity_per_service_list_head_index =
1449           nat_affinity_get_per_service_list_head_index ();
1450       else
1451         m->affinity_per_service_list_head_index = ~0;
1452
1453       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1454                    m - sm->static_mappings);
1455       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1456         {
1457           nat_elog_err ("static_mapping_by_external key add failed");
1458           return VNET_API_ERROR_UNSPECIFIED;
1459         }
1460
1461       for (i = 0; i < vec_len (locals); i++)
1462         {
1463           locals[i].fib_index =
1464             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1465                                                locals[i].vrf_id,
1466                                                sm->fib_src_low);
1467           if (!out2in_only)
1468             {
1469               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1470                            locals[i].fib_index, m->proto,
1471                            m - sm->static_mappings);
1472               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1473             }
1474           locals[i].prefix = (i == 0) ? locals[i].probability :
1475             (locals[i - 1].prefix + locals[i].probability);
1476           pool_get (m->locals, local);
1477           *local = locals[i];
1478           if (sm->num_workers > 1)
1479             {
1480               ip4_header_t ip = {
1481                 .src_address = locals[i].addr,
1482               };
1483               bitmap =
1484                 clib_bitmap_set (bitmap,
1485                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1486                                  1);
1487             }
1488         }
1489
1490       /* Assign workers */
1491       if (sm->num_workers > 1)
1492         {
1493           /* *INDENT-OFF* */
1494           clib_bitmap_foreach (i, bitmap,
1495             ({
1496                vec_add1(m->workers, i);
1497             }));
1498           /* *INDENT-ON* */
1499         }
1500     }
1501   else
1502     {
1503       if (!m)
1504         return VNET_API_ERROR_NO_SUCH_ENTRY;
1505
1506       if (!is_lb_static_mapping (m))
1507         return VNET_API_ERROR_INVALID_VALUE;
1508
1509       /* Free external address port */
1510       if (!(sm->static_mapping_only || out2in_only))
1511         {
1512           for (i = 0; i < vec_len (sm->addresses); i++)
1513             {
1514               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1515                 {
1516                   a = sm->addresses + i;
1517                   switch (proto)
1518                     {
1519 #define _(N, j, n, s) \
1520                     case NAT_PROTOCOL_##N: \
1521                       --a->busy_##n##_port_refcounts[e_port]; \
1522                       if (e_port > 1024) \
1523                         { \
1524                           a->busy_##n##_ports--; \
1525                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1526                         } \
1527                       break;
1528                       foreach_nat_protocol
1529 #undef _
1530                     default:
1531                       nat_elog_info ("unknown protocol");
1532                       return VNET_API_ERROR_INVALID_VALUE_2;
1533                     }
1534                   break;
1535                 }
1536             }
1537         }
1538
1539       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1540       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1541         {
1542           nat_elog_err ("static_mapping_by_external key del failed");
1543           return VNET_API_ERROR_UNSPECIFIED;
1544         }
1545
1546       /* *INDENT-OFF* */
1547       pool_foreach (local, m->locals,
1548       ({
1549           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1550                             sm->fib_src_low);
1551           if (!out2in_only)
1552             {
1553 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1554               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1555                 {
1556                   nat_elog_err ("static_mapping_by_local key del failed");
1557                   return VNET_API_ERROR_UNSPECIFIED;
1558                 }
1559             }
1560
1561           if (sm->num_workers > 1)
1562             {
1563               ip4_header_t ip = {
1564                 .src_address = local->addr,
1565               };
1566               tsm = vec_elt_at_index (sm->per_thread_data,
1567                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1568             }
1569           else
1570             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1571
1572           /* Delete sessions */
1573           pool_foreach (s, tsm->sessions, {
1574             if (!(is_lb_session (s)))
1575               continue;
1576
1577             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1578                 s->in2out.port != local->port)
1579               continue;
1580
1581             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1582             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1583           });
1584       }));
1585       /* *INDENT-ON* */
1586       if (m->affinity)
1587         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1588       pool_free (m->locals);
1589       vec_free (m->tag);
1590       vec_free (m->workers);
1591
1592       pool_put (sm->static_mappings, m);
1593     }
1594
1595   return 0;
1596 }
1597
1598 int
1599 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1600                                        ip4_address_t l_addr, u16 l_port,
1601                                        nat_protocol_t proto, u32 vrf_id,
1602                                        u8 probability, u8 is_add)
1603 {
1604   snat_main_t *sm = &snat_main;
1605   snat_static_mapping_t *m = 0;
1606   clib_bihash_kv_8_8_t kv, value;
1607   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1608   snat_main_per_thread_data_t *tsm;
1609   snat_session_t *s;
1610   u32 *locals = 0;
1611   uword *bitmap = 0;
1612   int i;
1613
1614   if (!sm->endpoint_dependent)
1615     return VNET_API_ERROR_FEATURE_DISABLED;
1616
1617   init_nat_k (&kv, e_addr, e_port, 0, proto);
1618   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1619     m = pool_elt_at_index (sm->static_mappings, value.value);
1620
1621   if (!m)
1622     return VNET_API_ERROR_NO_SUCH_ENTRY;
1623
1624   if (!is_lb_static_mapping (m))
1625     return VNET_API_ERROR_INVALID_VALUE;
1626
1627   /* *INDENT-OFF* */
1628   pool_foreach (local, m->locals,
1629   ({
1630     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1631         (local->vrf_id == vrf_id))
1632       {
1633         match_local = local;
1634         break;
1635       }
1636   }));
1637   /* *INDENT-ON* */
1638
1639   if (is_add)
1640     {
1641       if (match_local)
1642         return VNET_API_ERROR_VALUE_EXIST;
1643
1644       pool_get (m->locals, local);
1645       clib_memset (local, 0, sizeof (*local));
1646       local->addr.as_u32 = l_addr.as_u32;
1647       local->port = l_port;
1648       local->probability = probability;
1649       local->vrf_id = vrf_id;
1650       local->fib_index =
1651         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1652                                            sm->fib_src_low);
1653
1654       if (!is_out2in_only_static_mapping (m))
1655         {
1656           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto,
1657                        m - sm->static_mappings);
1658           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1659             nat_elog_err ("static_mapping_by_local key add failed");
1660         }
1661     }
1662   else
1663     {
1664       if (!match_local)
1665         return VNET_API_ERROR_NO_SUCH_ENTRY;
1666
1667       if (pool_elts (m->locals) < 3)
1668         return VNET_API_ERROR_UNSPECIFIED;
1669
1670       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1671                         sm->fib_src_low);
1672
1673       if (!is_out2in_only_static_mapping (m))
1674         {
1675           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1676           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1677             nat_elog_err ("static_mapping_by_local key del failed");
1678         }
1679
1680       if (sm->num_workers > 1)
1681         {
1682           ip4_header_t ip = {
1683             .src_address = local->addr,
1684           };
1685           tsm = vec_elt_at_index (sm->per_thread_data,
1686                                   sm->worker_in2out_cb (&ip, m->fib_index,
1687                                                         0));
1688         }
1689       else
1690         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1691
1692       /* Delete sessions */
1693       /* *INDENT-OFF* */
1694       pool_foreach (s, tsm->sessions, {
1695         if (!(is_lb_session (s)))
1696           continue;
1697
1698         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1699             s->in2out.port != match_local->port)
1700           continue;
1701
1702         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1703         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1704       });
1705       /* *INDENT-ON* */
1706
1707       pool_put (m->locals, match_local);
1708     }
1709
1710   vec_free (m->workers);
1711
1712   /* *INDENT-OFF* */
1713   pool_foreach (local, m->locals,
1714   ({
1715     vec_add1 (locals, local - m->locals);
1716     if (sm->num_workers > 1)
1717       {
1718         ip4_header_t ip;
1719         ip.src_address.as_u32 = local->addr.as_u32,
1720         bitmap = clib_bitmap_set (bitmap,
1721                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1722                                   1);
1723       }
1724   }));
1725   /* *INDENT-ON* */
1726
1727   ASSERT (vec_len (locals) > 1);
1728
1729   local = pool_elt_at_index (m->locals, locals[0]);
1730   local->prefix = local->probability;
1731   for (i = 1; i < vec_len (locals); i++)
1732     {
1733       local = pool_elt_at_index (m->locals, locals[i]);
1734       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1735       local->prefix = local->probability + prev_local->prefix;
1736     }
1737
1738   /* Assign workers */
1739   if (sm->num_workers > 1)
1740     {
1741       /* *INDENT-OFF* */
1742       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1743       /* *INDENT-ON* */
1744     }
1745
1746   return 0;
1747 }
1748
1749 int
1750 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1751                   u8 twice_nat)
1752 {
1753   snat_address_t *a = 0;
1754   snat_session_t *ses;
1755   u32 *ses_to_be_removed = 0, *ses_index;
1756   snat_main_per_thread_data_t *tsm;
1757   snat_static_mapping_t *m;
1758   snat_interface_t *interface;
1759   int i;
1760   snat_address_t *addresses =
1761     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1762
1763   /* Find SNAT address */
1764   for (i = 0; i < vec_len (addresses); i++)
1765     {
1766       if (addresses[i].addr.as_u32 == addr.as_u32)
1767         {
1768           a = addresses + i;
1769           break;
1770         }
1771     }
1772   if (!a)
1773     {
1774       nat_log_err ("no such address");
1775       return VNET_API_ERROR_NO_SUCH_ENTRY;
1776     }
1777
1778   if (delete_sm)
1779     {
1780       ip4_address_t pool_addr = { 0 };
1781       /* *INDENT-OFF* */
1782       pool_foreach (m, sm->static_mappings,
1783       ({
1784           if (m->external_addr.as_u32 == addr.as_u32)
1785             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1786                                             m->local_port, m->external_port,
1787                                             m->vrf_id,
1788                                             is_addr_only_static_mapping(m), ~0,
1789                                             m->proto, 0 /* is_add */,
1790                                             m->twice_nat,
1791                                             is_out2in_only_static_mapping(m),
1792                                             m->tag,
1793                                             is_identity_static_mapping(m),
1794                                             pool_addr, 0);
1795       }));
1796       /* *INDENT-ON* */
1797     }
1798   else
1799     {
1800       /* Check if address is used in some static mapping */
1801       if (is_snat_address_used_in_static_mapping (sm, addr))
1802         {
1803           nat_log_err ("address used in static mapping");
1804           return VNET_API_ERROR_UNSPECIFIED;
1805         }
1806     }
1807
1808   if (a->fib_index != ~0)
1809     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1810
1811   /* Delete sessions using address */
1812   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1813     {
1814       vec_foreach (tsm, sm->per_thread_data)
1815       {
1816         /* *INDENT-OFF* */
1817         pool_foreach (ses, tsm->sessions, ({
1818           if (ses->out2in.addr.as_u32 == addr.as_u32)
1819             {
1820               nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1821               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1822             }
1823         }));
1824         /* *INDENT-ON* */
1825
1826         if (sm->endpoint_dependent)
1827           {
1828             vec_foreach (ses_index, ses_to_be_removed)
1829             {
1830               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1831               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1832             }
1833           }
1834         else
1835           {
1836             vec_foreach (ses_index, ses_to_be_removed)
1837             {
1838               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1839               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1840             }
1841           }
1842
1843         vec_free (ses_to_be_removed);
1844       }
1845     }
1846
1847 #define _(N, i, n, s) \
1848   vec_free (a->busy_##n##_ports_per_thread);
1849   foreach_nat_protocol
1850 #undef _
1851     if (twice_nat)
1852     {
1853       vec_del1 (sm->twice_nat_addresses, i);
1854       return 0;
1855     }
1856   else
1857     vec_del1 (sm->addresses, i);
1858
1859   /* Delete external address from FIB */
1860   /* *INDENT-OFF* */
1861   pool_foreach (interface, sm->interfaces,
1862   ({
1863     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1864       continue;
1865
1866     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1867     break;
1868   }));
1869   pool_foreach (interface, sm->output_feature_interfaces,
1870   ({
1871     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1872       continue;
1873
1874     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1875     break;
1876   }));
1877   /* *INDENT-ON* */
1878
1879   return 0;
1880 }
1881
1882 static void
1883 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1884 {
1885 #define _(x)                                                                  \
1886   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1887                                 sw_if_index);                                 \
1888   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1889   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1890                                 sw_if_index);                                 \
1891   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1892   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1893                                 sw_if_index);                                 \
1894   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1895   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1896                                 sw_if_index);                                 \
1897   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1898   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1899                                 sw_if_index);                                 \
1900   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1901   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1902                                 sw_if_index);                                 \
1903   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1904   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1905                                 sw_if_index);                                 \
1906   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1907   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1908                                 sw_if_index);                                 \
1909   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1910   foreach_nat_counter;
1911 #undef _
1912   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1913   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1914 }
1915
1916 void
1917 expire_per_vrf_sessions (u32 fib_index)
1918 {
1919   per_vrf_sessions_t *per_vrf_sessions;
1920   snat_main_per_thread_data_t *tsm;
1921   snat_main_t *sm = &snat_main;
1922
1923   /* *INDENT-OFF* */
1924   vec_foreach (tsm, sm->per_thread_data)
1925     {
1926       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1927         {
1928           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1929               (per_vrf_sessions->tx_fib_index == fib_index))
1930             {
1931               per_vrf_sessions->expired = 1;
1932             }
1933         }
1934     }
1935   /* *INDENT-ON* */
1936 }
1937
1938 void
1939 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1940 {
1941   snat_main_t *sm = &snat_main;
1942   nat_fib_t *fib;
1943
1944   // we don't care if it is outside/inside fib
1945   // we just care about their ref_count
1946   // if it reaches 0 sessions should expire
1947   // because the fib isn't valid for NAT anymore
1948
1949   vec_foreach (fib, sm->fibs)
1950   {
1951     if (fib->fib_index == fib_index)
1952       {
1953         if (is_del)
1954           {
1955             fib->ref_count--;
1956             if (!fib->ref_count)
1957               {
1958                 vec_del1 (sm->fibs, fib - sm->fibs);
1959                 expire_per_vrf_sessions (fib_index);
1960               }
1961             return;
1962           }
1963         else
1964           fib->ref_count++;
1965       }
1966   }
1967   if (!is_del)
1968     {
1969       vec_add2 (sm->fibs, fib, 1);
1970       fib->ref_count = 1;
1971       fib->fib_index = fib_index;
1972     }
1973 }
1974
1975 int
1976 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1977 {
1978   snat_main_t *sm = &snat_main;
1979   snat_interface_t *i;
1980   const char *feature_name, *del_feature_name;
1981   snat_address_t *ap;
1982   snat_static_mapping_t *m;
1983   nat_outside_fib_t *outside_fib;
1984   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1985                                                        sw_if_index);
1986
1987   if (!sm->enabled)
1988     {
1989       nat_log_err ("nat44 is disabled");
1990       return VNET_API_ERROR_UNSUPPORTED;
1991     }
1992
1993   if (sm->out2in_dpo && !is_inside)
1994     {
1995       nat_log_err ("error unsupported");
1996       return VNET_API_ERROR_UNSUPPORTED;
1997     }
1998
1999   /* *INDENT-OFF* */
2000   pool_foreach (i, sm->output_feature_interfaces,
2001   ({
2002     if (i->sw_if_index == sw_if_index)
2003       {
2004         nat_log_err ("error interface already configured");
2005         return VNET_API_ERROR_VALUE_EXIST;
2006       }
2007   }));
2008   /* *INDENT-ON* */
2009
2010   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2011     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
2012   else
2013     {
2014       if (sm->num_workers > 1)
2015         feature_name =
2016           is_inside ? "nat44-in2out-worker-handoff" :
2017           "nat44-out2in-worker-handoff";
2018       else if (sm->endpoint_dependent)
2019         {
2020           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
2021         }
2022       else
2023         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
2024     }
2025
2026   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
2027     sm->fq_in2out_index =
2028       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
2029
2030   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2031     sm->fq_out2in_index =
2032       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
2033
2034   if (sm->endpoint_dependent)
2035     update_per_vrf_sessions_vec (fib_index, is_del);
2036
2037   if (!is_inside)
2038     {
2039       /* *INDENT-OFF* */
2040       vec_foreach (outside_fib, sm->outside_fibs)
2041         {
2042           if (outside_fib->fib_index == fib_index)
2043             {
2044               if (is_del)
2045                 {
2046                   outside_fib->refcount--;
2047                   if (!outside_fib->refcount)
2048                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2049                 }
2050               else
2051                 outside_fib->refcount++;
2052               goto feature_set;
2053             }
2054         }
2055       /* *INDENT-ON* */
2056       if (!is_del)
2057         {
2058           vec_add2 (sm->outside_fibs, outside_fib, 1);
2059           outside_fib->refcount = 1;
2060           outside_fib->fib_index = fib_index;
2061         }
2062     }
2063
2064 feature_set:
2065   /* *INDENT-OFF* */
2066   pool_foreach (i, sm->interfaces,
2067   ({
2068     if (i->sw_if_index == sw_if_index)
2069       {
2070         if (is_del)
2071           {
2072             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
2073               {
2074                 if (is_inside)
2075                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
2076                 else
2077                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
2078
2079                 if (sm->num_workers > 1)
2080                   {
2081                     del_feature_name = "nat44-handoff-classify";
2082                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2083                                                  "nat44-out2in-worker-handoff";
2084                   }
2085                 else if (sm->endpoint_dependent)
2086                   {
2087                     del_feature_name = "nat44-ed-classify";
2088                     feature_name = !is_inside ?  "nat-pre-in2out" :
2089                                                  "nat-pre-out2in";
2090                   }
2091                 else
2092                   {
2093                     del_feature_name = "nat44-classify";
2094                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2095                   }
2096
2097                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2098                 if (rv)
2099                   return rv;
2100                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2101                                              sw_if_index, 0, 0, 0);
2102                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2103                                              sw_if_index, 1, 0, 0);
2104                 if (!is_inside)
2105                   {
2106                     if (sm->endpoint_dependent)
2107                       vnet_feature_enable_disable ("ip4-local",
2108                                                    "nat44-ed-hairpinning",
2109                                                    sw_if_index, 1, 0, 0);
2110                     else
2111                       vnet_feature_enable_disable ("ip4-local",
2112                                                    "nat44-hairpinning",
2113                                                    sw_if_index, 1, 0, 0);
2114                   }
2115               }
2116             else
2117               {
2118                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2119                 if (rv)
2120                   return rv;
2121                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2122                                              sw_if_index, 0, 0, 0);
2123                 pool_put (sm->interfaces, i);
2124                 if (is_inside)
2125                   {
2126                     if (sm->endpoint_dependent)
2127                       vnet_feature_enable_disable ("ip4-local",
2128                                                    "nat44-ed-hairpinning",
2129                                                    sw_if_index, 0, 0, 0);
2130                     else
2131                       vnet_feature_enable_disable ("ip4-local",
2132                                                    "nat44-hairpinning",
2133                                                    sw_if_index, 0, 0, 0);
2134                   }
2135               }
2136           }
2137         else
2138           {
2139             if ((nat_interface_is_inside(i) && is_inside) ||
2140                 (nat_interface_is_outside(i) && !is_inside))
2141               return 0;
2142
2143             if (sm->num_workers > 1)
2144               {
2145                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2146                                                  "nat44-out2in-worker-handoff";
2147                 feature_name = "nat44-handoff-classify";
2148               }
2149             else if (sm->endpoint_dependent)
2150               {
2151                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2152                                                  "nat-pre-out2in";
2153
2154                 feature_name = "nat44-ed-classify";
2155               }
2156             else
2157               {
2158                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2159                 feature_name = "nat44-classify";
2160               }
2161
2162             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2163             if (rv)
2164               return rv;
2165             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2166                                          sw_if_index, 0, 0, 0);
2167             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2168                                          sw_if_index, 1, 0, 0);
2169             if (!is_inside)
2170               {
2171                 if (sm->endpoint_dependent)
2172                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2173                                                sw_if_index, 0, 0, 0);
2174                 else
2175                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2176                                                sw_if_index, 0, 0, 0);
2177               }
2178             goto set_flags;
2179           }
2180
2181         goto fib;
2182       }
2183   }));
2184   /* *INDENT-ON* */
2185
2186   if (is_del)
2187     {
2188       nat_log_err ("error interface couldn't be found");
2189       return VNET_API_ERROR_NO_SUCH_ENTRY;
2190     }
2191
2192   pool_get (sm->interfaces, i);
2193   i->sw_if_index = sw_if_index;
2194   i->flags = 0;
2195   nat_validate_counters (sm, sw_if_index);
2196
2197   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2198                                0);
2199
2200   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2201   if (rv)
2202     return rv;
2203
2204   if (is_inside && !sm->out2in_dpo)
2205     {
2206       if (sm->endpoint_dependent)
2207         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2208                                      sw_if_index, 1, 0, 0);
2209       else
2210         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2211                                      sw_if_index, 1, 0, 0);
2212     }
2213
2214 set_flags:
2215   if (is_inside)
2216     {
2217       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2218       return 0;
2219     }
2220   else
2221     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2222
2223   /* Add/delete external addresses to FIB */
2224 fib:
2225   /* *INDENT-OFF* */
2226   vec_foreach (ap, sm->addresses)
2227     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2228
2229   pool_foreach (m, sm->static_mappings,
2230   ({
2231     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2232       continue;
2233
2234     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2235   }));
2236   /* *INDENT-ON* */
2237
2238   return 0;
2239 }
2240
2241 int
2242 snat_interface_add_del_output_feature (u32 sw_if_index,
2243                                        u8 is_inside, int is_del)
2244 {
2245   snat_main_t *sm = &snat_main;
2246   snat_interface_t *i;
2247   snat_address_t *ap;
2248   snat_static_mapping_t *m;
2249   nat_outside_fib_t *outside_fib;
2250   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2251                                                        sw_if_index);
2252
2253   if (!sm->enabled)
2254     {
2255       nat_log_err ("nat44 is disabled");
2256       return VNET_API_ERROR_UNSUPPORTED;
2257     }
2258
2259   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2260     {
2261       nat_log_err ("error unsupported");
2262       return VNET_API_ERROR_UNSUPPORTED;
2263     }
2264
2265   /* *INDENT-OFF* */
2266   pool_foreach (i, sm->interfaces,
2267   ({
2268     if (i->sw_if_index == sw_if_index)
2269       {
2270         nat_log_err ("error interface already configured");
2271         return VNET_API_ERROR_VALUE_EXIST;
2272       }
2273   }));
2274   /* *INDENT-ON* */
2275
2276   if (sm->endpoint_dependent)
2277     update_per_vrf_sessions_vec (fib_index, is_del);
2278
2279   if (!is_inside)
2280     {
2281       /* *INDENT-OFF* */
2282       vec_foreach (outside_fib, sm->outside_fibs)
2283         {
2284           if (outside_fib->fib_index == fib_index)
2285             {
2286               if (is_del)
2287                 {
2288                   outside_fib->refcount--;
2289                   if (!outside_fib->refcount)
2290                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2291                 }
2292               else
2293                 outside_fib->refcount++;
2294               goto feature_set;
2295             }
2296         }
2297       /* *INDENT-ON* */
2298       if (!is_del)
2299         {
2300           vec_add2 (sm->outside_fibs, outside_fib, 1);
2301           outside_fib->refcount = 1;
2302           outside_fib->fib_index = fib_index;
2303         }
2304     }
2305
2306 feature_set:
2307   if (is_inside)
2308     {
2309       if (sm->endpoint_dependent)
2310         {
2311           int rv =
2312             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2313           if (rv)
2314             return rv;
2315           rv =
2316             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2317                                                             !is_del);
2318           if (rv)
2319             return rv;
2320           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2321                                        sw_if_index, !is_del, 0, 0);
2322           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2323                                        sw_if_index, !is_del, 0, 0);
2324         }
2325       else
2326         {
2327           int rv =
2328             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2329           if (rv)
2330             return rv;
2331           rv =
2332             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2333                                                             !is_del);
2334           if (rv)
2335             return rv;
2336           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2337                                        sw_if_index, !is_del, 0, 0);
2338           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2339                                        sw_if_index, !is_del, 0, 0);
2340         }
2341       goto fq;
2342     }
2343
2344   if (sm->num_workers > 1)
2345     {
2346       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2347       if (rv)
2348         return rv;
2349       rv =
2350         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2351       if (rv)
2352         return rv;
2353       vnet_feature_enable_disable ("ip4-unicast",
2354                                    "nat44-out2in-worker-handoff",
2355                                    sw_if_index, !is_del, 0, 0);
2356       vnet_feature_enable_disable ("ip4-output",
2357                                    "nat44-in2out-output-worker-handoff",
2358                                    sw_if_index, !is_del, 0, 0);
2359     }
2360   else
2361     {
2362       if (sm->endpoint_dependent)
2363         {
2364           int rv =
2365             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2366           if (rv)
2367             return rv;
2368           rv =
2369             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2370                                                             !is_del);
2371           if (rv)
2372             return rv;
2373           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2374                                        sw_if_index, !is_del, 0, 0);
2375           vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2376                                        sw_if_index, !is_del, 0, 0);
2377         }
2378       else
2379         {
2380           int rv =
2381             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2382           if (rv)
2383             return rv;
2384           rv =
2385             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2386                                                             !is_del);
2387           if (rv)
2388             return rv;
2389           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2390                                        sw_if_index, !is_del, 0, 0);
2391           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2392                                        sw_if_index, !is_del, 0, 0);
2393         }
2394     }
2395
2396 fq:
2397   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2398     sm->fq_in2out_output_index =
2399       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2400
2401   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2402     sm->fq_out2in_index =
2403       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2404
2405   /* *INDENT-OFF* */
2406   pool_foreach (i, sm->output_feature_interfaces,
2407   ({
2408     if (i->sw_if_index == sw_if_index)
2409       {
2410         if (is_del)
2411           pool_put (sm->output_feature_interfaces, i);
2412         else
2413           return VNET_API_ERROR_VALUE_EXIST;
2414
2415         goto fib;
2416       }
2417   }));
2418   /* *INDENT-ON* */
2419
2420   if (is_del)
2421     {
2422       nat_log_err ("error interface couldn't be found");
2423       return VNET_API_ERROR_NO_SUCH_ENTRY;
2424     }
2425
2426   pool_get (sm->output_feature_interfaces, i);
2427   i->sw_if_index = sw_if_index;
2428   i->flags = 0;
2429   nat_validate_counters (sm, sw_if_index);
2430   if (is_inside)
2431     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2432   else
2433     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2434
2435   /* Add/delete external addresses to FIB */
2436 fib:
2437   if (is_inside)
2438     return 0;
2439
2440   /* *INDENT-OFF* */
2441   vec_foreach (ap, sm->addresses)
2442     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2443
2444   pool_foreach (m, sm->static_mappings,
2445   ({
2446     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2447       continue;
2448
2449     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2450   }));
2451   /* *INDENT-ON* */
2452
2453   return 0;
2454 }
2455
2456 int
2457 snat_set_workers (uword * bitmap)
2458 {
2459   snat_main_t *sm = &snat_main;
2460   int i, j = 0;
2461
2462   if (sm->num_workers < 2)
2463     return VNET_API_ERROR_FEATURE_DISABLED;
2464
2465   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2466     return VNET_API_ERROR_INVALID_WORKER;
2467
2468   vec_free (sm->workers);
2469   /* *INDENT-OFF* */
2470   clib_bitmap_foreach (i, bitmap,
2471     ({
2472       vec_add1(sm->workers, i);
2473       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2474       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2475       j++;
2476     }));
2477   /* *INDENT-ON* */
2478
2479   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2480
2481   return 0;
2482 }
2483
2484 static void
2485 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2486                          u32 sw_if_index, u32 new_fib_index,
2487                          u32 old_fib_index)
2488 {
2489   snat_main_t *sm = &snat_main;
2490   nat_outside_fib_t *outside_fib;
2491   snat_interface_t *i;
2492   u8 is_add = 1;
2493   u8 match = 0;
2494
2495   if (!sm->enabled || (new_fib_index == old_fib_index)
2496       || (!vec_len (sm->outside_fibs)))
2497     {
2498       return;
2499     }
2500
2501   /* *INDENT-OFF* */
2502   pool_foreach (i, sm->interfaces,
2503     ({
2504       if (i->sw_if_index == sw_if_index)
2505         {
2506           if (!(nat_interface_is_outside (i)))
2507             return;
2508           match = 1;
2509         }
2510     }));
2511
2512   pool_foreach (i, sm->output_feature_interfaces,
2513     ({
2514       if (i->sw_if_index == sw_if_index)
2515         {
2516           if (!(nat_interface_is_outside (i)))
2517             return;
2518           match = 1;
2519         }
2520     }));
2521   /* *INDENT-ON* */
2522
2523   if (!match)
2524     return;
2525
2526   vec_foreach (outside_fib, sm->outside_fibs)
2527   {
2528     if (outside_fib->fib_index == old_fib_index)
2529       {
2530         outside_fib->refcount--;
2531         if (!outside_fib->refcount)
2532           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2533         break;
2534       }
2535   }
2536
2537   vec_foreach (outside_fib, sm->outside_fibs)
2538   {
2539     if (outside_fib->fib_index == new_fib_index)
2540       {
2541         outside_fib->refcount++;
2542         is_add = 0;
2543         break;
2544       }
2545   }
2546
2547   if (is_add)
2548     {
2549       vec_add2 (sm->outside_fibs, outside_fib, 1);
2550       outside_fib->refcount = 1;
2551       outside_fib->fib_index = new_fib_index;
2552     }
2553 }
2554
2555 static void
2556 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2557                          u32 sw_if_index, u32 new_fib_index,
2558                          u32 old_fib_index);
2559
2560 static void
2561 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2562                                        uword opaque,
2563                                        u32 sw_if_index,
2564                                        ip4_address_t * address,
2565                                        u32 address_length,
2566                                        u32 if_address_index, u32 is_delete);
2567
2568 static void
2569 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2570                                  uword opaque,
2571                                  u32 sw_if_index,
2572                                  ip4_address_t * address,
2573                                  u32 address_length,
2574                                  u32 if_address_index, u32 is_delete);
2575
2576 static int
2577 nat_alloc_addr_and_port_default (snat_address_t * addresses, u32 fib_index,
2578                                  u32 thread_index, nat_protocol_t proto,
2579                                  ip4_address_t * addr, u16 * port,
2580                                  u16 port_per_thread, u32 snat_thread_index);
2581
2582 void
2583 test_key_calc_split ()
2584 {
2585   ip4_address_t l_addr;
2586   l_addr.as_u8[0] = 1;
2587   l_addr.as_u8[1] = 1;
2588   l_addr.as_u8[2] = 1;
2589   l_addr.as_u8[3] = 1;
2590   ip4_address_t r_addr;
2591   r_addr.as_u8[0] = 2;
2592   r_addr.as_u8[1] = 2;
2593   r_addr.as_u8[2] = 2;
2594   r_addr.as_u8[3] = 2;
2595   u16 l_port = 40001;
2596   u16 r_port = 40301;
2597   u8 proto = 9;
2598   u32 fib_index = 9000001;
2599   u32 thread_index = 3000000001;
2600   u32 session_index = 3000000221;
2601   clib_bihash_kv_16_8_t kv;
2602   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2603               thread_index, session_index);
2604   ip4_address_t l_addr2;
2605   ip4_address_t r_addr2;
2606   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2607   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2608   u16 l_port2 = 0;
2609   u16 r_port2 = 0;
2610   u8 proto2 = 0;
2611   u32 fib_index2 = 0;
2612   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2613                &r_port2);
2614   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2615   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2616   ASSERT (l_port == l_port2);
2617   ASSERT (r_port == r_port2);
2618   ASSERT (proto == proto2);
2619   ASSERT (fib_index == fib_index2);
2620   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2621   ASSERT (session_index == ed_value_get_session_index (&kv));
2622
2623   fib_index = 7001;
2624   proto = 5;
2625   nat_protocol_t proto3 = ~0;
2626   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2627   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2628   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2629   ASSERT (l_port == l_port2);
2630   ASSERT (proto == proto3);
2631   ASSERT (fib_index == fib_index2);
2632 }
2633
2634 static clib_error_t *
2635 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2636 {
2637   snat_main_t *sm = &snat_main;
2638   u32 fib_index;
2639
2640   if (sm->endpoint_dependent)
2641     {
2642       // TODO: consider removing all NAT interfaces
2643       if (!is_add)
2644         {
2645           fib_index = ip4_fib_index_from_table_id (table_id);
2646           if (fib_index != ~0)
2647             expire_per_vrf_sessions (fib_index);
2648         }
2649     }
2650   return 0;
2651 }
2652
2653 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2654
2655 void
2656 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2657 {
2658   vlib_node_t *node;
2659
2660   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2661   sm->ei_out2in_node_index = node->index;
2662   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2663   sm->ei_in2out_node_index = node->index;
2664   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2665   sm->ei_in2out_output_node_index = node->index;
2666
2667   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2668   sm->ed_out2in_node_index = node->index;
2669   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2670   sm->ed_in2out_node_index = node->index;
2671   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2672   sm->ed_in2out_output_node_index = node->index;
2673
2674   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2675   sm->error_node_index = node->index;
2676   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2677   sm->pre_in2out_node_index = node->index;
2678   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2679   sm->pre_out2in_node_index = node->index;
2680   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2681   sm->pre_in2out_node_index = node->index;
2682   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2683   sm->pre_out2in_node_index = node->index;
2684   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2685   sm->in2out_fast_node_index = node->index;
2686   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2687   sm->in2out_slowpath_node_index = node->index;
2688   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2689   sm->in2out_slowpath_output_node_index = node->index;
2690   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2691   sm->ed_in2out_slowpath_node_index = node->index;
2692   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2693   sm->out2in_fast_node_index = node->index;
2694   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2695   sm->ed_out2in_slowpath_node_index = node->index;
2696   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2697   sm->hairpinning_node_index = node->index;
2698   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2699   sm->hairpin_dst_node_index = node->index;
2700   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2701   sm->hairpin_src_node_index = node->index;
2702   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2703   sm->ed_hairpinning_node_index = node->index;
2704   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2705   sm->ed_hairpin_dst_node_index = node->index;
2706   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2707   sm->ed_hairpin_src_node_index = node->index;
2708 }
2709
2710 #define nat_init_simple_counter(c, n, sn) \
2711 do                                        \
2712   {                                       \
2713     c.name = n;                           \
2714     c.stat_segment_name = sn;             \
2715     vlib_validate_simple_counter (&c, 0); \
2716     vlib_zero_simple_counter (&c, 0);     \
2717   } while (0);
2718
2719 static clib_error_t *
2720 nat_init (vlib_main_t * vm)
2721 {
2722   snat_main_t *sm = &snat_main;
2723   vlib_thread_main_t *tm = vlib_get_thread_main ();
2724   vlib_thread_registration_t *tr;
2725   ip4_add_del_interface_address_callback_t cbi = { 0 };
2726   ip4_table_bind_callback_t cbt = { 0 };
2727   u32 i, num_threads = 0;
2728   uword *p, *bitmap = 0;
2729
2730   clib_memset (sm, 0, sizeof (*sm));
2731
2732   // required
2733   sm->vnet_main = vnet_get_main ();
2734   // convenience
2735   sm->ip4_main = &ip4_main;
2736   sm->api_main = vlibapi_get_main ();
2737   sm->ip4_lookup_main = &ip4_main.lookup_main;
2738
2739   // frame queue indices used for handoff
2740   sm->fq_out2in_index = ~0;
2741   sm->fq_in2out_index = ~0;
2742   sm->fq_in2out_output_index = ~0;
2743
2744   sm->log_level = SNAT_LOG_ERROR;
2745
2746   nat44_set_node_indexes (sm, vm);
2747   sm->log_class = vlib_log_register_class ("nat", 0);
2748   nat_ipfix_logging_init (vm);
2749
2750   nat_init_simple_counter (sm->total_users, "total-users",
2751                            "/nat44/total-users");
2752   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2753                            "/nat44/total-sessions");
2754   nat_init_simple_counter (sm->user_limit_reached, "user-limit-reached",
2755                            "/nat44/user-limit-reached");
2756
2757 #define _(x)                                            \
2758   sm->counters.fastpath.in2out.x.name = #x;             \
2759   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2760       "/nat44/in2out/fastpath/" #x;                     \
2761   sm->counters.slowpath.in2out.x.name = #x;             \
2762   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2763       "/nat44/in2out/slowpath/" #x;                     \
2764   sm->counters.fastpath.out2in.x.name = #x;             \
2765   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2766       "/nat44/out2in/fastpath/" #x;                     \
2767   sm->counters.slowpath.out2in.x.name = #x;             \
2768   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2769       "/nat44/out2in/slowpath/" #x;                     \
2770   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2771   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2772       "/nat44/ed/in2out/fastpath/" #x;                  \
2773   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2774   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2775       "/nat44/ed/in2out/slowpath/" #x;                  \
2776   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2777   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2778       "/nat44/ed/out2in/fastpath/" #x;                  \
2779   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2780   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2781       "/nat44/ed/out2in/slowpath/" #x;
2782   foreach_nat_counter;
2783 #undef _
2784   sm->counters.hairpinning.name = "hairpinning";
2785   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2786
2787   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2788   if (p)
2789     {
2790       tr = (vlib_thread_registration_t *) p[0];
2791       if (tr)
2792         {
2793           sm->num_workers = tr->count;
2794           sm->first_worker_index = tr->first_index;
2795         }
2796     }
2797   num_threads = tm->n_vlib_mains - 1;
2798   sm->port_per_thread = 0xffff - 1024;
2799   vec_validate (sm->per_thread_data, num_threads);
2800
2801   /* Use all available workers by default */
2802   if (sm->num_workers > 1)
2803     {
2804
2805       for (i = 0; i < sm->num_workers; i++)
2806         bitmap = clib_bitmap_set (bitmap, i, 1);
2807       snat_set_workers (bitmap);
2808       clib_bitmap_free (bitmap);
2809     }
2810   else
2811     sm->per_thread_data[0].snat_thread_index = 0;
2812
2813   /* callbacks to call when interface address changes. */
2814   cbi.function = snat_ip4_add_del_interface_address_cb;
2815   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2816   cbi.function = nat_ip4_add_del_addr_only_sm_cb;
2817   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2818
2819   /* callbacks to call when interface to table biding changes */
2820   cbt.function = snat_update_outside_fib;
2821   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2822
2823   sm->fib_src_low =
2824     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2825                          FIB_SOURCE_BH_SIMPLE);
2826   sm->fib_src_hi =
2827     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2828                          FIB_SOURCE_BH_SIMPLE);
2829
2830   /* used only by out2in-dpo feature */
2831   nat_dpo_module_init ();
2832
2833   nat_affinity_init (vm);
2834   nat_ha_init (vm, sm->num_workers, num_threads);
2835
2836   test_key_calc_split ();
2837   return nat44_api_hookup (vm);
2838 }
2839
2840 VLIB_INIT_FUNCTION (nat_init);
2841
2842 int
2843 nat44_plugin_enable (nat44_config_t c)
2844 {
2845   snat_main_t *sm = &snat_main;
2846   u32 static_mapping_buckets = 1024;
2847   u32 static_mapping_memory_size = 64 << 20;
2848
2849   if (sm->enabled)
2850     {
2851       nat_log_err ("nat44 is enabled");
2852       return 1;
2853     }
2854
2855   // c.static_mapping_only + c.connection_tracking
2856   //  - supported in NAT EI & NAT ED
2857   // c.out2in_dpo, c.static_mapping_only
2858   //  - supported in NAT EI
2859
2860   if (c.endpoint_dependent)
2861     {
2862       if ((c.static_mapping_only && !c.connection_tracking) || c.out2in_dpo)
2863         {
2864           nat_log_err ("unsupported combination of configuration");
2865           return 1;
2866         }
2867       if (c.users || c.user_sessions)
2868         {
2869           nat_log_err ("unsupported combination of configuration");
2870           return 1;
2871         }
2872     }
2873
2874   // reset to defaults:
2875   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2876   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2877   //
2878   nat_reset_timeouts (&sm->timeouts);
2879
2880   // nat44 feature configuration
2881   sm->endpoint_dependent = c.endpoint_dependent;
2882   sm->static_mapping_only = c.static_mapping_only;
2883   sm->static_mapping_connection_tracking = c.connection_tracking;
2884   sm->forwarding_enabled = 0;
2885   sm->mss_clamping = 0;
2886
2887   if (!c.users)
2888     c.users = 1024;
2889
2890   sm->max_users_per_thread = c.users;
2891   sm->user_buckets = nat_calc_bihash_buckets (c.users);
2892
2893   if (!c.sessions)
2894     c.sessions = 10 * 1024;
2895
2896   sm->max_translations_per_thread = c.sessions;
2897   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2898
2899   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2900   sm->max_translations_per_user
2901     = c.user_sessions ? c.user_sessions : sm->max_translations_per_thread;
2902
2903   sm->outside_vrf_id = c.outside_vrf;
2904   sm->outside_fib_index =
2905     fib_table_find_or_create_and_lock
2906     (FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2907
2908   sm->inside_vrf_id = c.inside_vrf;
2909   sm->inside_fib_index =
2910     fib_table_find_or_create_and_lock
2911     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2912
2913   if (c.endpoint_dependent)
2914     {
2915       sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2916       sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
2917       sm->out2in_node_index = sm->ed_out2in_node_index;
2918       sm->in2out_node_index = sm->ed_in2out_node_index;
2919       sm->in2out_output_node_index = sm->ed_in2out_output_node_index;
2920       sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
2921       sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
2922
2923       // try to move it into nat44_db_init,
2924       // consider static mapping requirements
2925       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
2926                              sm->translation_buckets, 0);
2927       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
2928                                           format_ed_session_kvp);
2929
2930
2931       nat_affinity_enable ();
2932
2933       nat_ha_enable (nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb, nat_ha_sref_ed_cb);
2934     }
2935   else
2936     {
2937       sm->worker_out2in_cb = snat_get_worker_out2in_cb;
2938       sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2939       sm->out2in_node_index = sm->ei_out2in_node_index;
2940       sm->in2out_node_index = sm->ei_in2out_node_index;
2941       sm->in2out_output_node_index = sm->ei_in2out_output_node_index;
2942       sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
2943       sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
2944
2945       nat_ha_enable (nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
2946     }
2947
2948   // c.static_mapping & c.connection_tracking require
2949   // session database
2950   if (!c.static_mapping_only
2951       || (c.static_mapping_only && c.connection_tracking))
2952     {
2953       snat_main_per_thread_data_t *tsm;
2954       /* *INDENT-OFF* */
2955       vec_foreach (tsm, sm->per_thread_data)
2956         {
2957           nat44_db_init (tsm);
2958         }
2959       /* *INDENT-ON* */
2960     }
2961   else
2962     {
2963       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2964       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2965     }
2966
2967   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
2968                         "static_mapping_by_local", static_mapping_buckets,
2969                         static_mapping_memory_size);
2970   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
2971                                      format_static_mapping_kvp);
2972
2973   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
2974                         "static_mapping_by_external",
2975                         static_mapping_buckets, static_mapping_memory_size);
2976   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
2977                                      format_static_mapping_kvp);
2978
2979   // last: reset counters
2980   vlib_zero_simple_counter (&sm->total_users, 0);
2981   vlib_zero_simple_counter (&sm->total_sessions, 0);
2982   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2983
2984   sm->enabled = 1;
2985   sm->rconfig = c;
2986
2987   return 0;
2988 }
2989
2990 void
2991 nat44_addresses_free (snat_address_t ** addresses)
2992 {
2993   snat_address_t *ap;
2994   /* *INDENT-OFF* */
2995   vec_foreach (ap, *addresses)
2996     {
2997     #define _(N, i, n, s) \
2998       vec_free (ap->busy_##n##_ports_per_thread);
2999       foreach_nat_protocol
3000     #undef _
3001     }
3002   /* *INDENT-ON* */
3003   vec_free (*addresses);
3004   *addresses = 0;
3005 }
3006
3007 int
3008 nat44_plugin_disable ()
3009 {
3010   snat_main_t *sm = &snat_main;
3011   snat_interface_t *i, *vec;
3012   int error = 0;
3013
3014   if (!sm->enabled)
3015     {
3016       nat_log_err ("nat44 is disabled");
3017       return 1;
3018     }
3019
3020   // first unregister all nodes from interfaces
3021   vec = vec_dup (sm->interfaces);
3022   /* *INDENT-OFF* */
3023   vec_foreach (i, vec)
3024     {
3025       if (nat_interface_is_inside(i))
3026         error = snat_interface_add_del (i->sw_if_index, 1, 1);
3027       if (nat_interface_is_outside(i))
3028         error = snat_interface_add_del (i->sw_if_index, 0, 1);
3029
3030       if (error)
3031         {
3032           nat_log_err ("error occurred while removing interface %u",
3033                        i->sw_if_index);
3034         }
3035     }
3036   /* *INDENT-ON* */
3037   vec_free (vec);
3038   sm->interfaces = 0;
3039
3040   vec = vec_dup (sm->output_feature_interfaces);
3041   /* *INDENT-OFF* */
3042   vec_foreach (i, vec)
3043     {
3044       if (nat_interface_is_inside(i))
3045         error = snat_interface_add_del_output_feature (i->sw_if_index, 1, 1);
3046       if (nat_interface_is_outside(i))
3047         error = snat_interface_add_del_output_feature (i->sw_if_index, 0, 1);
3048
3049       if (error)
3050         {
3051           nat_log_err ("error occurred while removing interface %u",
3052                        i->sw_if_index);
3053         }
3054     }
3055   /* *INDENT-ON* */
3056   vec_free (vec);
3057   sm->output_feature_interfaces = 0;
3058
3059   vec_free (sm->max_translations_per_fib);
3060
3061   if (sm->endpoint_dependent)
3062     {
3063       nat_affinity_disable ();
3064       clib_bihash_free_16_8 (&sm->out2in_ed);
3065     }
3066
3067   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
3068   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
3069
3070   if (!sm->static_mapping_only ||
3071       (sm->static_mapping_only && sm->static_mapping_connection_tracking))
3072     {
3073       snat_main_per_thread_data_t *tsm;
3074      /* *INDENT-OFF* */
3075       vec_foreach (tsm, sm->per_thread_data)
3076         {
3077           nat44_db_free (tsm);
3078         }
3079       /* *INDENT-ON* */
3080     }
3081
3082   pool_free (sm->static_mappings);
3083
3084   nat44_addresses_free (&sm->addresses);
3085   nat44_addresses_free (&sm->twice_nat_addresses);
3086
3087
3088   vec_free (sm->to_resolve);
3089   vec_free (sm->auto_add_sw_if_indices);
3090   vec_free (sm->auto_add_sw_if_indices_twice_nat);
3091
3092   sm->to_resolve = 0;
3093   sm->auto_add_sw_if_indices = 0;
3094   sm->auto_add_sw_if_indices_twice_nat = 0;
3095
3096   sm->forwarding_enabled = 0;
3097
3098   sm->enabled = 0;
3099   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
3100
3101   return 0;
3102 }
3103
3104 void
3105 snat_free_outside_address_and_port (snat_address_t * addresses,
3106                                     u32 thread_index,
3107                                     ip4_address_t * addr,
3108                                     u16 port, nat_protocol_t protocol)
3109 {
3110   snat_address_t *a;
3111   u32 address_index;
3112   u16 port_host_byte_order = clib_net_to_host_u16 (port);
3113
3114   for (address_index = 0; address_index < vec_len (addresses);
3115        address_index++)
3116     {
3117       if (addresses[address_index].addr.as_u32 == addr->as_u32)
3118         break;
3119     }
3120
3121   ASSERT (address_index < vec_len (addresses));
3122
3123   a = addresses + address_index;
3124
3125   switch (protocol)
3126     {
3127 #define _(N, i, n, s) \
3128     case NAT_PROTOCOL_##N: \
3129       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
3130       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
3131       a->busy_##n##_ports--; \
3132       a->busy_##n##_ports_per_thread[thread_index]--; \
3133       break;
3134       foreach_nat_protocol
3135 #undef _
3136     default:
3137       nat_elog_info ("unknown protocol");
3138       return;
3139     }
3140 }
3141
3142 static int
3143 nat_set_outside_address_and_port (snat_address_t * addresses,
3144                                   u32 thread_index, ip4_address_t addr,
3145                                   u16 port, nat_protocol_t protocol)
3146 {
3147   snat_address_t *a = 0;
3148   u32 address_index;
3149   u16 port_host_byte_order = clib_net_to_host_u16 (port);
3150
3151   for (address_index = 0; address_index < vec_len (addresses);
3152        address_index++)
3153     {
3154       if (addresses[address_index].addr.as_u32 != addr.as_u32)
3155         continue;
3156
3157       a = addresses + address_index;
3158       switch (protocol)
3159         {
3160 #define _(N, j, n, s) \
3161         case NAT_PROTOCOL_##N: \
3162           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
3163             return VNET_API_ERROR_INSTANCE_IN_USE; \
3164           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
3165           a->busy_##n##_ports_per_thread[thread_index]++; \
3166           a->busy_##n##_ports++; \
3167           return 0;
3168           foreach_nat_protocol
3169 #undef _
3170         default:
3171           nat_elog_info ("unknown protocol");
3172           return 1;
3173         }
3174     }
3175
3176   return VNET_API_ERROR_NO_SUCH_ENTRY;
3177 }
3178
3179 int
3180 snat_static_mapping_match (snat_main_t * sm,
3181                            ip4_address_t match_addr,
3182                            u16 match_port,
3183                            u32 match_fib_index,
3184                            nat_protocol_t match_protocol,
3185                            ip4_address_t * mapping_addr,
3186                            u16 * mapping_port,
3187                            u32 * mapping_fib_index,
3188                            u8 by_external,
3189                            u8 * is_addr_only,
3190                            twice_nat_type_t * twice_nat,
3191                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
3192                            u8 * is_identity_nat, snat_static_mapping_t ** out)
3193 {
3194   clib_bihash_kv_8_8_t kv, value;
3195   clib_bihash_8_8_t *mapping_hash;
3196   snat_static_mapping_t *m;
3197   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
3198   nat44_lb_addr_port_t *local;
3199   u8 backend_index;
3200
3201   if (!by_external)
3202     {
3203       mapping_hash = &sm->static_mapping_by_local;
3204       init_nat_k (&kv, match_addr, match_port, match_fib_index,
3205                   match_protocol);
3206       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3207         {
3208           /* Try address only mapping */
3209           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
3210           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3211             return 1;
3212         }
3213     }
3214   else
3215     {
3216       mapping_hash = &sm->static_mapping_by_external;
3217       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
3218       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3219         {
3220           /* Try address only mapping */
3221           init_nat_k (&kv, match_addr, 0, 0, 0);
3222           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3223             return 1;
3224         }
3225     }
3226
3227   m = pool_elt_at_index (sm->static_mappings, value.value);
3228
3229   if (by_external)
3230     {
3231       if (is_lb_static_mapping (m))
3232         {
3233           if (PREDICT_FALSE (lb != 0))
3234             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
3235           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
3236                                                           match_addr,
3237                                                           match_protocol,
3238                                                           match_port,
3239                                                           &backend_index))
3240             {
3241               local = pool_elt_at_index (m->locals, backend_index);
3242               *mapping_addr = local->addr;
3243               *mapping_port = local->port;
3244               *mapping_fib_index = local->fib_index;
3245               goto end;
3246             }
3247           // pick locals matching this worker
3248           if (PREDICT_FALSE (sm->num_workers > 1))
3249             {
3250               u32 thread_index = vlib_get_thread_index ();
3251               /* *INDENT-OFF* */
3252               pool_foreach_index (i, m->locals,
3253               ({
3254                 local = pool_elt_at_index (m->locals, i);
3255
3256                 ip4_header_t ip = {
3257                   .src_address = local->addr,
3258                 };
3259
3260                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
3261                     thread_index)
3262                   {
3263                     vec_add1 (tmp, i);
3264                   }
3265               }));
3266               /* *INDENT-ON* */
3267               ASSERT (vec_len (tmp) != 0);
3268             }
3269           else
3270             {
3271               /* *INDENT-OFF* */
3272               pool_foreach_index (i, m->locals,
3273               ({
3274                 vec_add1 (tmp, i);
3275               }));
3276               /* *INDENT-ON* */
3277             }
3278           hi = vec_len (tmp) - 1;
3279           local = pool_elt_at_index (m->locals, tmp[hi]);
3280           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
3281           while (lo < hi)
3282             {
3283               mid = ((hi - lo) >> 1) + lo;
3284               local = pool_elt_at_index (m->locals, tmp[mid]);
3285               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
3286             }
3287           local = pool_elt_at_index (m->locals, tmp[lo]);
3288           if (!(local->prefix >= rand))
3289             return 1;
3290           *mapping_addr = local->addr;
3291           *mapping_port = local->port;
3292           *mapping_fib_index = local->fib_index;
3293           if (m->affinity)
3294             {
3295               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
3296                                                 match_protocol, match_port,
3297                                                 tmp[lo], m->affinity,
3298                                                 m->affinity_per_service_list_head_index))
3299                 nat_elog_info ("create affinity record failed");
3300             }
3301           vec_free (tmp);
3302         }
3303       else
3304         {
3305           if (PREDICT_FALSE (lb != 0))
3306             *lb = NO_LB_NAT;
3307           *mapping_fib_index = m->fib_index;
3308           *mapping_addr = m->local_addr;
3309           /* Address only mapping doesn't change port */
3310           *mapping_port = is_addr_only_static_mapping (m) ? match_port
3311             : m->local_port;
3312         }
3313     }
3314   else
3315     {
3316       *mapping_addr = m->external_addr;
3317       /* Address only mapping doesn't change port */
3318       *mapping_port = is_addr_only_static_mapping (m) ? match_port
3319         : m->external_port;
3320       *mapping_fib_index = sm->outside_fib_index;
3321     }
3322
3323 end:
3324   if (PREDICT_FALSE (is_addr_only != 0))
3325     *is_addr_only = is_addr_only_static_mapping (m);
3326
3327   if (PREDICT_FALSE (twice_nat != 0))
3328     *twice_nat = m->twice_nat;
3329
3330   if (PREDICT_FALSE (is_identity_nat != 0))
3331     *is_identity_nat = is_identity_static_mapping (m);
3332
3333   if (out != 0)
3334     *out = m;
3335
3336   return 0;
3337 }
3338
3339 int
3340 snat_alloc_outside_address_and_port (snat_address_t * addresses,
3341                                      u32 fib_index,
3342                                      u32 thread_index,
3343                                      nat_protocol_t proto,
3344                                      ip4_address_t * addr,
3345                                      u16 * port,
3346                                      u16 port_per_thread,
3347                                      u32 snat_thread_index)
3348 {
3349   snat_main_t *sm = &snat_main;
3350
3351   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, proto,
3352                                   addr, port, port_per_thread,
3353                                   snat_thread_index);
3354 }
3355
3356 static int
3357 nat_alloc_addr_and_port_default (snat_address_t * addresses,
3358                                  u32 fib_index,
3359                                  u32 thread_index,
3360                                  nat_protocol_t proto,
3361                                  ip4_address_t * addr,
3362                                  u16 * port,
3363                                  u16 port_per_thread, u32 snat_thread_index)
3364 {
3365   int i;
3366   snat_address_t *a, *ga = 0;
3367   u32 portnum;
3368
3369   for (i = 0; i < vec_len (addresses); i++)
3370     {
3371       a = addresses + i;
3372       switch (proto)
3373         {
3374 #define _(N, j, n, s) \
3375         case NAT_PROTOCOL_##N: \
3376           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
3377             { \
3378               if (a->fib_index == fib_index) \
3379                 { \
3380                   while (1) \
3381                     { \
3382                       portnum = (port_per_thread * \
3383                         snat_thread_index) + \
3384                         snat_random_port(0, port_per_thread - 1) + 1024; \
3385                       if (a->busy_##n##_port_refcounts[portnum]) \
3386                         continue; \
3387                       --a->busy_##n##_port_refcounts[portnum]; \
3388                       a->busy_##n##_ports_per_thread[thread_index]++; \
3389                       a->busy_##n##_ports++; \
3390                       *addr = a->addr; \
3391                       *port = clib_host_to_net_u16(portnum); \
3392                       return 0; \
3393                     } \
3394                 } \
3395               else if (a->fib_index == ~0) \
3396                 { \
3397                   ga = a; \
3398                 } \
3399             } \
3400           break;
3401           foreach_nat_protocol
3402 #undef _
3403         default:
3404           nat_elog_info ("unknown protocol");
3405           return 1;
3406         }
3407
3408     }
3409
3410   if (ga)
3411     {
3412       a = ga;
3413       switch (proto)
3414         {
3415 #define _(N, j, n, s) \
3416         case NAT_PROTOCOL_##N: \
3417           while (1) \
3418             { \
3419               portnum = (port_per_thread * \
3420                 snat_thread_index) + \
3421                 snat_random_port(0, port_per_thread - 1) + 1024; \
3422               if (a->busy_##n##_port_refcounts[portnum]) \
3423                 continue; \
3424               ++a->busy_##n##_port_refcounts[portnum]; \
3425               a->busy_##n##_ports_per_thread[thread_index]++; \
3426               a->busy_##n##_ports++; \
3427               *addr = a->addr; \
3428               *port = clib_host_to_net_u16(portnum); \
3429               return 0; \
3430             }
3431           break;
3432           foreach_nat_protocol
3433 #undef _
3434         default:
3435           nat_elog_info ("unknown protocol");
3436           return 1;
3437         }
3438     }
3439
3440   /* Totally out of translations to use... */
3441   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
3442   return 1;
3443 }
3444
3445 static int
3446 nat_alloc_addr_and_port_mape (snat_address_t * addresses, u32 fib_index,
3447                               u32 thread_index, nat_protocol_t proto,
3448                               ip4_address_t * addr, u16 * port,
3449                               u16 port_per_thread, u32 snat_thread_index)
3450 {
3451   snat_main_t *sm = &snat_main;
3452   snat_address_t *a = addresses;
3453   u16 m, ports, portnum, A, j;
3454   m = 16 - (sm->psid_offset + sm->psid_length);
3455   ports = (1 << (16 - sm->psid_length)) - (1 << m);
3456
3457   if (!vec_len (addresses))
3458     goto exhausted;
3459
3460   switch (proto)
3461     {
3462 #define _(N, i, n, s) \
3463     case NAT_PROTOCOL_##N: \
3464       if (a->busy_##n##_ports < ports) \
3465         { \
3466           while (1) \
3467             { \
3468               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3469               j = snat_random_port(0, pow2_mask(m)); \
3470               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3471               if (a->busy_##n##_port_refcounts[portnum]) \
3472                 continue; \
3473               ++a->busy_##n##_port_refcounts[portnum]; \
3474               a->busy_##n##_ports++; \
3475               *addr = a->addr; \
3476               *port = clib_host_to_net_u16 (portnum); \
3477               return 0; \
3478             } \
3479         } \
3480       break;
3481       foreach_nat_protocol
3482 #undef _
3483     default:
3484       nat_elog_info ("unknown protocol");
3485       return 1;
3486     }
3487
3488 exhausted:
3489   /* Totally out of translations to use... */
3490   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
3491   return 1;
3492 }
3493
3494 static int
3495 nat_alloc_addr_and_port_range (snat_address_t * addresses, u32 fib_index,
3496                                u32 thread_index, nat_protocol_t proto,
3497                                ip4_address_t * addr, u16 * port,
3498                                u16 port_per_thread, u32 snat_thread_index)
3499 {
3500   snat_main_t *sm = &snat_main;
3501   snat_address_t *a = addresses;
3502   u16 portnum, ports;
3503
3504   ports = sm->end_port - sm->start_port + 1;
3505
3506   if (!vec_len (addresses))
3507     goto exhausted;
3508
3509   switch (proto)
3510     {
3511 #define _(N, i, n, s) \
3512     case NAT_PROTOCOL_##N: \
3513       if (a->busy_##n##_ports < ports) \
3514         { \
3515           while (1) \
3516             { \
3517               portnum = snat_random_port(sm->start_port, sm->end_port); \
3518               if (a->busy_##n##_port_refcounts[portnum]) \
3519                 continue; \
3520               ++a->busy_##n##_port_refcounts[portnum]; \
3521               a->busy_##n##_ports++; \
3522               *addr = a->addr; \
3523               *port = clib_host_to_net_u16 (portnum); \
3524               return 0; \
3525             } \
3526         } \
3527       break;
3528       foreach_nat_protocol
3529 #undef _
3530     default:
3531       nat_elog_info ("unknown protocol");
3532       return 1;
3533     }
3534
3535 exhausted:
3536   /* Totally out of translations to use... */
3537   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
3538   return 1;
3539 }
3540
3541 void
3542 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3543 {
3544   snat_main_t *sm = &snat_main;
3545   dpo_id_t dpo_v4 = DPO_INVALID;
3546   fib_prefix_t pfx = {
3547     .fp_proto = FIB_PROTOCOL_IP4,
3548     .fp_len = 32,
3549     .fp_addr.ip4.as_u32 = addr.as_u32,
3550   };
3551
3552   if (is_add)
3553     {
3554       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3555       fib_table_entry_special_dpo_add (0, &pfx, sm->fib_src_hi,
3556                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3557       dpo_reset (&dpo_v4);
3558     }
3559   else
3560     {
3561       fib_table_entry_special_remove (0, &pfx, sm->fib_src_hi);
3562     }
3563 }
3564
3565 static u32
3566 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3567                            u8 is_output)
3568 {
3569   snat_main_t *sm = &snat_main;
3570   u32 next_worker_index = 0;
3571   u32 hash;
3572
3573   next_worker_index = sm->first_worker_index;
3574   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3575     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3576
3577   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3578     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3579   else
3580     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3581
3582   return next_worker_index;
3583 }
3584
3585 static u32
3586 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3587                            u32 rx_fib_index0, u8 is_output)
3588 {
3589   snat_main_t *sm = &snat_main;
3590   udp_header_t *udp;
3591   u16 port;
3592   clib_bihash_kv_8_8_t kv, value;
3593   snat_static_mapping_t *m;
3594   u32 proto;
3595   u32 next_worker_index = 0;
3596
3597   /* first try static mappings without port */
3598   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3599     {
3600       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
3601       if (!clib_bihash_search_8_8
3602           (&sm->static_mapping_by_external, &kv, &value))
3603         {
3604           m = pool_elt_at_index (sm->static_mappings, value.value);
3605           return m->workers[0];
3606         }
3607     }
3608
3609   proto = ip_proto_to_nat_proto (ip0->protocol);
3610   udp = ip4_next_header (ip0);
3611   port = udp->dst_port;
3612
3613   /* unknown protocol */
3614   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3615     {
3616       /* use current thread */
3617       return vlib_get_thread_index ();
3618     }
3619
3620   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3621     {
3622       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3623       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3624       if (!icmp_type_is_error_message
3625           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3626         port = vnet_buffer (b)->ip.reass.l4_src_port;
3627       else
3628         {
3629           /* if error message, then it's not fragmented and we can access it */
3630           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3631           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3632           void *l4_header = ip4_next_header (inner_ip);
3633           switch (proto)
3634             {
3635             case NAT_PROTOCOL_ICMP:
3636               icmp = (icmp46_header_t *) l4_header;
3637               echo = (icmp_echo_header_t *) (icmp + 1);
3638               port = echo->identifier;
3639               break;
3640             case NAT_PROTOCOL_UDP:
3641             case NAT_PROTOCOL_TCP:
3642               port = ((tcp_udp_header_t *) l4_header)->src_port;
3643               break;
3644             default:
3645               return vlib_get_thread_index ();
3646             }
3647         }
3648     }
3649
3650   /* try static mappings with port */
3651   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3652     {
3653       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
3654       if (!clib_bihash_search_8_8
3655           (&sm->static_mapping_by_external, &kv, &value))
3656         {
3657           m = pool_elt_at_index (sm->static_mappings, value.value);
3658           return m->workers[0];
3659         }
3660     }
3661
3662   /* worker by outside port */
3663   next_worker_index = sm->first_worker_index;
3664   next_worker_index +=
3665     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3666   return next_worker_index;
3667 }
3668
3669 static u32
3670 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3671                                u8 is_output)
3672 {
3673   snat_main_t *sm = &snat_main;
3674   u32 next_worker_index = sm->first_worker_index;
3675   u32 hash;
3676
3677   clib_bihash_kv_16_8_t kv16, value16;
3678   snat_main_per_thread_data_t *tsm;
3679   udp_header_t *udp;
3680
3681   if (PREDICT_FALSE (is_output))
3682     {
3683       u32 fib_index = sm->outside_fib_index;
3684       nat_outside_fib_t *outside_fib;
3685       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3686       fib_prefix_t pfx = {
3687         .fp_proto = FIB_PROTOCOL_IP4,
3688         .fp_len = 32,
3689         .fp_addr = {
3690                     .ip4.as_u32 = ip->dst_address.as_u32,
3691                     }
3692         ,
3693       };
3694
3695       udp = ip4_next_header (ip);
3696
3697       switch (vec_len (sm->outside_fibs))
3698         {
3699         case 0:
3700           fib_index = sm->outside_fib_index;
3701           break;
3702         case 1:
3703           fib_index = sm->outside_fibs[0].fib_index;
3704           break;
3705         default:
3706             /* *INDENT-OFF* */
3707             vec_foreach (outside_fib, sm->outside_fibs)
3708               {
3709                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3710                 if (FIB_NODE_INDEX_INVALID != fei)
3711                   {
3712                     if (fib_entry_get_resolving_interface (fei) != ~0)
3713                       {
3714                         fib_index = outside_fib->fib_index;
3715                         break;
3716                       }
3717                   }
3718               }
3719             /* *INDENT-ON* */
3720           break;
3721         }
3722
3723       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3724                  udp->dst_port, fib_index, ip->protocol);
3725
3726       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3727                                                   &kv16, &value16)))
3728         {
3729           tsm =
3730             vec_elt_at_index (sm->per_thread_data,
3731                               ed_value_get_thread_index (&value16));
3732           next_worker_index += tsm->thread_index;
3733
3734           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3735                                   next_worker_index, fib_index,
3736                                   clib_net_to_host_u32 (ip->
3737                                                         src_address.as_u32),
3738                                   clib_net_to_host_u32 (ip->
3739                                                         dst_address.as_u32));
3740
3741           return next_worker_index;
3742         }
3743     }
3744
3745   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3746     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3747
3748   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3749     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3750   else
3751     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3752
3753   if (PREDICT_TRUE (!is_output))
3754     {
3755       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3756                               next_worker_index, rx_fib_index,
3757                               clib_net_to_host_u32 (ip->src_address.as_u32),
3758                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3759     }
3760   else
3761     {
3762       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3763                               next_worker_index, rx_fib_index,
3764                               clib_net_to_host_u32 (ip->src_address.as_u32),
3765                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3766     }
3767
3768   return next_worker_index;
3769 }
3770
3771 static u32
3772 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3773                                u32 rx_fib_index, u8 is_output)
3774 {
3775   snat_main_t *sm = &snat_main;
3776   clib_bihash_kv_8_8_t kv, value;
3777   clib_bihash_kv_16_8_t kv16, value16;
3778   snat_main_per_thread_data_t *tsm;
3779
3780   u32 proto, next_worker_index = 0;
3781   udp_header_t *udp;
3782   u16 port;
3783   snat_static_mapping_t *m;
3784   u32 hash;
3785
3786   proto = ip_proto_to_nat_proto (ip->protocol);
3787
3788   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3789     {
3790       udp = ip4_next_header (ip);
3791
3792       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3793                  udp->src_port, rx_fib_index, ip->protocol);
3794
3795       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3796                                                   &kv16, &value16)))
3797         {
3798           tsm =
3799             vec_elt_at_index (sm->per_thread_data,
3800                               ed_value_get_thread_index (&value16));
3801           vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
3802             ed_value_get_session_index (&value16);
3803           next_worker_index = sm->first_worker_index + tsm->thread_index;
3804           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3805                                   next_worker_index, rx_fib_index,
3806                                   clib_net_to_host_u32 (ip->
3807                                                         src_address.as_u32),
3808                                   clib_net_to_host_u32 (ip->
3809                                                         dst_address.as_u32));
3810           return next_worker_index;
3811         }
3812     }
3813   else if (proto == NAT_PROTOCOL_ICMP)
3814     {
3815       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
3816         {
3817           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3818                                                       &kv16, &value16)))
3819             {
3820               tsm =
3821                 vec_elt_at_index (sm->per_thread_data,
3822                                   ed_value_get_thread_index (&value16));
3823               next_worker_index = sm->first_worker_index + tsm->thread_index;
3824               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3825                                       next_worker_index, rx_fib_index,
3826                                       clib_net_to_host_u32 (ip->
3827                                                             src_address.as_u32),
3828                                       clib_net_to_host_u32 (ip->
3829                                                             dst_address.as_u32));
3830               return next_worker_index;
3831             }
3832         }
3833     }
3834
3835   /* first try static mappings without port */
3836   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3837     {
3838       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3839       if (!clib_bihash_search_8_8
3840           (&sm->static_mapping_by_external, &kv, &value))
3841         {
3842           m = pool_elt_at_index (sm->static_mappings, value.value);
3843           next_worker_index = m->workers[0];
3844           goto done;
3845         }
3846     }
3847
3848   /* unknown protocol */
3849   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3850     {
3851       /* use current thread */
3852       next_worker_index = vlib_get_thread_index ();
3853       goto done;
3854     }
3855
3856   udp = ip4_next_header (ip);
3857   port = udp->dst_port;
3858
3859   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3860     {
3861       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3862       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3863       if (!icmp_type_is_error_message
3864           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3865         port = vnet_buffer (b)->ip.reass.l4_src_port;
3866       else
3867         {
3868           /* if error message, then it's not fragmented and we can access it */
3869           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3870           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3871           void *l4_header = ip4_next_header (inner_ip);
3872           switch (proto)
3873             {
3874             case NAT_PROTOCOL_ICMP:
3875               icmp = (icmp46_header_t *) l4_header;
3876               echo = (icmp_echo_header_t *) (icmp + 1);
3877               port = echo->identifier;
3878               break;
3879             case NAT_PROTOCOL_UDP:
3880             case NAT_PROTOCOL_TCP:
3881               port = ((tcp_udp_header_t *) l4_header)->src_port;
3882               break;
3883             default:
3884               next_worker_index = vlib_get_thread_index ();
3885               goto done;
3886             }
3887         }
3888     }
3889
3890   /* try static mappings with port */
3891   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3892     {
3893       init_nat_k (&kv, ip->dst_address, port, 0, proto);
3894       if (!clib_bihash_search_8_8
3895           (&sm->static_mapping_by_external, &kv, &value))
3896         {
3897           m = pool_elt_at_index (sm->static_mappings, value.value);
3898           if (!is_lb_static_mapping (m))
3899             {
3900               next_worker_index = m->workers[0];
3901               goto done;
3902             }
3903
3904           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3905             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3906
3907           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3908             next_worker_index =
3909               m->workers[hash & (_vec_len (m->workers) - 1)];
3910           else
3911             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3912           goto done;
3913         }
3914     }
3915
3916   /* worker by outside port */
3917   next_worker_index = sm->first_worker_index;
3918   next_worker_index +=
3919     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3920
3921 done:
3922   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3923                           clib_net_to_host_u32 (ip->src_address.as_u32),
3924                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3925   return next_worker_index;
3926 }
3927
3928 void
3929 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3930                 ip4_address_t * out_addr, u16 out_port,
3931                 ip4_address_t * eh_addr, u16 eh_port,
3932                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3933                 u32 fib_index, u16 flags, u32 thread_index)
3934 {
3935   snat_main_t *sm = &snat_main;
3936   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3937   snat_user_t *u;
3938   snat_session_t *s;
3939   clib_bihash_kv_8_8_t kv;
3940   vlib_main_t *vm = vlib_get_main ();
3941   f64 now = vlib_time_now (vm);
3942   nat_outside_fib_t *outside_fib;
3943   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3944   fib_prefix_t pfx = {
3945     .fp_proto = FIB_PROTOCOL_IP4,
3946     .fp_len = 32,
3947     .fp_addr = {
3948                 .ip4.as_u32 = eh_addr->as_u32,
3949                 },
3950   };
3951
3952   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3953     {
3954       if (nat_set_outside_address_and_port
3955           (sm->addresses, thread_index, *out_addr, out_port, proto))
3956         return;
3957     }
3958
3959   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3960   if (!u)
3961     return;
3962
3963   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3964   if (!s)
3965     return;
3966
3967   if (sm->endpoint_dependent)
3968     {
3969       nat_ed_lru_insert (tsm, s, now, nat_proto_to_ip_proto (proto));
3970     }
3971
3972   s->out2in.addr.as_u32 = out_addr->as_u32;
3973   s->out2in.port = out_port;
3974   s->nat_proto = proto;
3975   s->last_heard = now;
3976   s->flags = flags;
3977   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3978   s->ext_host_port = eh_port;
3979   user_session_increment (sm, u, snat_is_session_static (s));
3980   switch (vec_len (sm->outside_fibs))
3981     {
3982     case 0:
3983       s->out2in.fib_index = sm->outside_fib_index;
3984       break;
3985     case 1:
3986       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3987       break;
3988     default:
3989       /* *INDENT-OFF* */
3990       vec_foreach (outside_fib, sm->outside_fibs)
3991         {
3992           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3993           if (FIB_NODE_INDEX_INVALID != fei)
3994             {
3995               if (fib_entry_get_resolving_interface (fei) != ~0)
3996                 {
3997                   s->out2in.fib_index = outside_fib->fib_index;
3998                   break;
3999                 }
4000             }
4001         }
4002       /* *INDENT-ON* */
4003       break;
4004     }
4005   init_nat_o2i_kv (&kv, s, s - tsm->sessions);
4006   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
4007     nat_elog_warn ("out2in key add failed");
4008
4009   s->in2out.addr.as_u32 = in_addr->as_u32;
4010   s->in2out.port = in_port;
4011   s->in2out.fib_index = fib_index;
4012   init_nat_i2o_kv (&kv, s, s - tsm->sessions);
4013   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
4014     nat_elog_warn ("in2out key add failed");
4015 }
4016
4017 void
4018 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
4019                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
4020                 u32 ti)
4021 {
4022   snat_main_t *sm = &snat_main;
4023   clib_bihash_kv_8_8_t kv, value;
4024   u32 thread_index;
4025   snat_session_t *s;
4026   snat_main_per_thread_data_t *tsm;
4027
4028   if (sm->num_workers > 1)
4029     thread_index =
4030       sm->first_worker_index +
4031       (sm->workers[(clib_net_to_host_u16 (out_port) -
4032                     1024) / sm->port_per_thread]);
4033   else
4034     thread_index = sm->num_workers;
4035   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
4036
4037   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
4038   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
4039     return;
4040
4041   s = pool_elt_at_index (tsm->sessions, value.value);
4042   nat_free_session_data (sm, s, thread_index, 1);
4043   nat44_delete_session (sm, s, thread_index);
4044 }
4045
4046 void
4047 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
4048                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
4049                 u32 total_pkts, u64 total_bytes, u32 thread_index)
4050 {
4051   snat_main_t *sm = &snat_main;
4052   clib_bihash_kv_8_8_t kv, value;
4053   snat_session_t *s;
4054   snat_main_per_thread_data_t *tsm;
4055
4056   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
4057
4058   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
4059   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
4060     return;
4061
4062   s = pool_elt_at_index (tsm->sessions, value.value);
4063   s->total_pkts = total_pkts;
4064   s->total_bytes = total_bytes;
4065 }
4066
4067 void
4068 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
4069                    ip4_address_t * out_addr, u16 out_port,
4070                    ip4_address_t * eh_addr, u16 eh_port,
4071                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
4072                    u32 fib_index, u16 flags, u32 thread_index)
4073 {
4074   snat_main_t *sm = &snat_main;
4075   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
4076   snat_session_t *s;
4077   clib_bihash_kv_16_8_t kv;
4078   vlib_main_t *vm = vlib_get_main ();
4079   f64 now = vlib_time_now (vm);
4080   nat_outside_fib_t *outside_fib;
4081   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
4082   fib_prefix_t pfx = {
4083     .fp_proto = FIB_PROTOCOL_IP4,
4084     .fp_len = 32,
4085     .fp_addr = {
4086                 .ip4.as_u32 = eh_addr->as_u32,
4087                 },
4088   };
4089
4090
4091   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
4092     {
4093       if (nat_set_outside_address_and_port
4094           (sm->addresses, thread_index, *out_addr, out_port, proto))
4095         return;
4096     }
4097
4098   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
4099     {
4100       if (nat_set_outside_address_and_port
4101           (sm->addresses, thread_index, *ehn_addr, ehn_port, proto))
4102         return;
4103     }
4104
4105   s = nat_ed_session_alloc (sm, thread_index, now, proto);
4106   if (!s)
4107     return;
4108
4109   s->last_heard = now;
4110   s->flags = flags;
4111   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
4112   s->ext_host_nat_port = s->ext_host_port = eh_port;
4113   if (is_twice_nat_session (s))
4114     {
4115       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
4116       s->ext_host_nat_port = ehn_port;
4117     }
4118   switch (vec_len (sm->outside_fibs))
4119     {
4120     case 0:
4121       s->out2in.fib_index = sm->outside_fib_index;
4122       break;
4123     case 1:
4124       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
4125       break;
4126     default:
4127       /* *INDENT-OFF* */
4128       vec_foreach (outside_fib, sm->outside_fibs)
4129         {
4130           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
4131           if (FIB_NODE_INDEX_INVALID != fei)
4132             {
4133               if (fib_entry_get_resolving_interface (fei) != ~0)
4134                 {
4135                   s->out2in.fib_index = outside_fib->fib_index;
4136                   break;
4137                 }
4138             }
4139         }
4140       /* *INDENT-ON* */
4141       break;
4142     }
4143   s->nat_proto = proto;
4144   s->out2in.addr.as_u32 = out_addr->as_u32;
4145   s->out2in.port = out_port;
4146
4147   s->in2out.addr.as_u32 = in_addr->as_u32;
4148   s->in2out.port = in_port;
4149   s->in2out.fib_index = fib_index;
4150
4151   init_ed_kv (&kv, *in_addr, in_port, s->ext_host_nat_addr,
4152               s->ext_host_nat_port, fib_index, nat_proto_to_ip_proto (proto),
4153               thread_index, s - tsm->sessions);
4154   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
4155     nat_elog_warn ("in2out key add failed");
4156
4157   init_ed_kv (&kv, *out_addr, out_port, *eh_addr, eh_port,
4158               s->out2in.fib_index, nat_proto_to_ip_proto (proto),
4159               thread_index, s - tsm->sessions);
4160   if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 1))
4161     nat_elog_warn ("out2in key add failed");
4162 }
4163
4164 void
4165 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
4166                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4167                    u32 fib_index, u32 ti)
4168 {
4169   snat_main_t *sm = &snat_main;
4170   clib_bihash_kv_16_8_t kv, value;
4171   u32 thread_index;
4172   snat_session_t *s;
4173   snat_main_per_thread_data_t *tsm;
4174
4175   if (sm->num_workers > 1)
4176     thread_index =
4177       sm->first_worker_index +
4178       (sm->workers[(clib_net_to_host_u16 (out_port) -
4179                     1024) / sm->port_per_thread]);
4180   else
4181     thread_index = sm->num_workers;
4182   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
4183
4184   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
4185   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
4186     return;
4187
4188   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
4189   nat_free_session_data (sm, s, thread_index, 1);
4190   nat44_delete_session (sm, s, thread_index);
4191 }
4192
4193 void
4194 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
4195                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4196                    u32 fib_index, u32 total_pkts, u64 total_bytes,
4197                    u32 thread_index)
4198 {
4199   snat_main_t *sm = &snat_main;
4200   clib_bihash_kv_16_8_t kv, value;
4201   snat_session_t *s;
4202   snat_main_per_thread_data_t *tsm;
4203
4204   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
4205
4206   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
4207   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
4208     return;
4209
4210   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
4211   s->total_pkts = total_pkts;
4212   s->total_bytes = total_bytes;
4213 }
4214
4215 static u32
4216 nat_calc_bihash_buckets (u32 n_elts)
4217 {
4218   n_elts = n_elts / 2.5;
4219   u64 lower_pow2 = 1;
4220   while (lower_pow2 * 2 < n_elts)
4221     {
4222       lower_pow2 = 2 * lower_pow2;
4223     }
4224   u64 upper_pow2 = 2 * lower_pow2;
4225   if ((upper_pow2 - n_elts) < (n_elts - lower_pow2))
4226     {
4227       if (upper_pow2 <= UINT32_MAX)
4228         {
4229           return upper_pow2;
4230         }
4231     }
4232   return lower_pow2;
4233 }
4234
4235 u32
4236 nat44_get_max_session_limit ()
4237 {
4238   snat_main_t *sm = &snat_main;
4239   u32 max_limit = 0, len = 0;
4240
4241   for (; len < vec_len (sm->max_translations_per_fib); len++)
4242     {
4243       if (max_limit < sm->max_translations_per_fib[len])
4244         max_limit = sm->max_translations_per_fib[len];
4245     }
4246   return max_limit;
4247 }
4248
4249 int
4250 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
4251 {
4252   snat_main_t *sm = &snat_main;
4253   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4254   u32 len = vec_len (sm->max_translations_per_fib);
4255
4256   if (len <= fib_index)
4257     {
4258       vec_validate (sm->max_translations_per_fib, fib_index + 1);
4259
4260       for (; len < vec_len (sm->max_translations_per_fib); len++)
4261         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
4262     }
4263
4264   sm->max_translations_per_fib[fib_index] = session_limit;
4265   return 0;
4266 }
4267
4268 int
4269 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
4270 {
4271   snat_main_t *sm = &snat_main;
4272
4273   if (nat44_set_session_limit (session_limit, vrf_id))
4274     return 1;
4275   sm->max_translations_per_thread = nat44_get_max_session_limit ();
4276
4277   sm->translation_buckets =
4278     nat_calc_bihash_buckets (sm->max_translations_per_thread);
4279
4280   nat44_sessions_clear ();
4281   return 0;
4282 }
4283
4284 void
4285 nat44_db_init (snat_main_per_thread_data_t * tsm)
4286 {
4287   snat_main_t *sm = &snat_main;
4288
4289   pool_alloc (tsm->sessions, sm->max_translations_per_thread);
4290   pool_alloc (tsm->lru_pool, sm->max_translations_per_thread);
4291
4292   dlist_elt_t *head;
4293
4294   pool_get (tsm->lru_pool, head);
4295   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
4296   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
4297
4298   pool_get (tsm->lru_pool, head);
4299   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
4300   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
4301
4302   pool_get (tsm->lru_pool, head);
4303   tsm->udp_lru_head_index = head - tsm->lru_pool;
4304   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
4305
4306   pool_get (tsm->lru_pool, head);
4307   tsm->icmp_lru_head_index = head - tsm->lru_pool;
4308   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
4309
4310   pool_get (tsm->lru_pool, head);
4311   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
4312   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
4313
4314   if (sm->endpoint_dependent)
4315     {
4316       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
4317                              sm->translation_buckets, 0);
4318       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
4319                                           format_ed_session_kvp);
4320       /*
4321          clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4322          sm->translation_buckets, 0);
4323          clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4324          format_ed_session_kvp); */
4325     }
4326   else
4327     {
4328       clib_bihash_init_8_8 (&tsm->in2out, "in2out", sm->translation_buckets,
4329                             0);
4330       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
4331       clib_bihash_init_8_8 (&tsm->out2in, "out2in", sm->translation_buckets,
4332                             0);
4333       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
4334     }
4335
4336   // TODO: ED nat is not using these
4337   // before removal large refactor required
4338   pool_alloc (tsm->list_pool, sm->max_translations_per_thread);
4339   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets, 0);
4340   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
4341 }
4342
4343 void
4344 nat44_db_free (snat_main_per_thread_data_t * tsm)
4345 {
4346   snat_main_t *sm = &snat_main;
4347
4348   pool_free (tsm->sessions);
4349   pool_free (tsm->lru_pool);
4350
4351   if (sm->endpoint_dependent)
4352     {
4353       clib_bihash_free_16_8 (&tsm->in2out_ed);
4354       vec_free (tsm->per_vrf_sessions_vec);
4355     }
4356   else
4357     {
4358       clib_bihash_free_8_8 (&tsm->in2out);
4359       clib_bihash_free_8_8 (&tsm->out2in);
4360     }
4361
4362   // TODO: resolve static mappings (put only to !ED)
4363   pool_free (tsm->users);
4364   pool_free (tsm->list_pool);
4365   clib_bihash_free_8_8 (&tsm->user_hash);
4366 }
4367
4368 void
4369 nat44_sessions_clear ()
4370 {
4371   snat_main_t *sm = &snat_main;
4372   snat_main_per_thread_data_t *tsm;
4373
4374   if (sm->endpoint_dependent)
4375     {
4376       clib_bihash_free_16_8 (&sm->out2in_ed);
4377       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4378                              clib_max (1,
4379                                        sm->num_workers) *
4380                              sm->translation_buckets, 0);
4381       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4382                                           format_ed_session_kvp);
4383     }
4384
4385   /* *INDENT-OFF* */
4386   vec_foreach (tsm, sm->per_thread_data)
4387     {
4388       u32 ti;
4389
4390       nat44_db_free (tsm);
4391       nat44_db_init (tsm);
4392
4393       ti = tsm->snat_thread_index;
4394       vlib_set_simple_counter (&sm->total_users, ti, 0, 0);
4395       vlib_set_simple_counter (&sm->total_sessions, ti, 0, 0);
4396     }
4397   /* *INDENT-ON* */
4398 }
4399
4400 static void
4401 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4402                                  uword opaque,
4403                                  u32 sw_if_index,
4404                                  ip4_address_t * address,
4405                                  u32 address_length,
4406                                  u32 if_address_index, u32 is_delete)
4407 {
4408   snat_main_t *sm = &snat_main;
4409   snat_static_map_resolve_t *rp;
4410   snat_static_mapping_t *m;
4411   clib_bihash_kv_8_8_t kv, value;
4412   int i, rv;
4413   ip4_address_t l_addr;
4414
4415   if (!sm->enabled)
4416     return;
4417
4418   for (i = 0; i < vec_len (sm->to_resolve); i++)
4419     {
4420       rp = sm->to_resolve + i;
4421       if (rp->addr_only == 0)
4422         continue;
4423       if (rp->sw_if_index == sw_if_index)
4424         goto match;
4425     }
4426
4427   return;
4428
4429 match:
4430   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
4431               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
4432   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4433     m = 0;
4434   else
4435     m = pool_elt_at_index (sm->static_mappings, value.value);
4436
4437   if (!is_delete)
4438     {
4439       /* Don't trip over lease renewal, static config */
4440       if (m)
4441         return;
4442     }
4443   else
4444     {
4445       if (!m)
4446         return;
4447     }
4448
4449   /* Indetity mapping? */
4450   if (rp->l_addr.as_u32 == 0)
4451     l_addr.as_u32 = address[0].as_u32;
4452   else
4453     l_addr.as_u32 = rp->l_addr.as_u32;
4454   /* Add the static mapping */
4455   rv = snat_add_static_mapping (l_addr,
4456                                 address[0],
4457                                 rp->l_port,
4458                                 rp->e_port,
4459                                 rp->vrf_id,
4460                                 rp->addr_only, ~0 /* sw_if_index */ ,
4461                                 rp->proto, !is_delete, rp->twice_nat,
4462                                 rp->out2in_only, rp->tag, rp->identity_nat,
4463                                 rp->pool_addr, rp->exact);
4464   if (rv)
4465     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4466 }
4467
4468 static void
4469 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4470                                        uword opaque,
4471                                        u32 sw_if_index,
4472                                        ip4_address_t * address,
4473                                        u32 address_length,
4474                                        u32 if_address_index, u32 is_delete)
4475 {
4476   snat_main_t *sm = &snat_main;
4477   snat_static_map_resolve_t *rp;
4478   ip4_address_t l_addr;
4479   int i, j;
4480   int rv;
4481   u8 twice_nat = 0;
4482   snat_address_t *addresses = sm->addresses;
4483
4484   if (!sm->enabled)
4485     return;
4486
4487   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4488     {
4489       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4490         goto match;
4491     }
4492
4493   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4494     {
4495       twice_nat = 1;
4496       addresses = sm->twice_nat_addresses;
4497       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4498         goto match;
4499     }
4500
4501   return;
4502
4503 match:
4504   if (!is_delete)
4505     {
4506       /* Don't trip over lease renewal, static config */
4507       for (j = 0; j < vec_len (addresses); j++)
4508         if (addresses[j].addr.as_u32 == address->as_u32)
4509           return;
4510
4511       (void) snat_add_address (sm, address, ~0, twice_nat);
4512       /* Scan static map resolution vector */
4513       for (j = 0; j < vec_len (sm->to_resolve); j++)
4514         {
4515           rp = sm->to_resolve + j;
4516           if (rp->addr_only)
4517             continue;
4518           /* On this interface? */
4519           if (rp->sw_if_index == sw_if_index)
4520             {
4521               /* Indetity mapping? */
4522               if (rp->l_addr.as_u32 == 0)
4523                 l_addr.as_u32 = address[0].as_u32;
4524               else
4525                 l_addr.as_u32 = rp->l_addr.as_u32;
4526               /* Add the static mapping */
4527               rv = snat_add_static_mapping (l_addr,
4528                                             address[0],
4529                                             rp->l_port,
4530                                             rp->e_port,
4531                                             rp->vrf_id,
4532                                             rp->addr_only,
4533                                             ~0 /* sw_if_index */ ,
4534                                             rp->proto,
4535                                             rp->is_add, rp->twice_nat,
4536                                             rp->out2in_only, rp->tag,
4537                                             rp->identity_nat,
4538                                             rp->pool_addr, rp->exact);
4539               if (rv)
4540                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4541                                     "i4", rv);
4542             }
4543         }
4544       return;
4545     }
4546   else
4547     {
4548       (void) snat_del_address (sm, address[0], 1, twice_nat);
4549       return;
4550     }
4551 }
4552
4553 int
4554 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4555                             u8 twice_nat)
4556 {
4557   ip4_main_t *ip4_main = sm->ip4_main;
4558   ip4_address_t *first_int_addr;
4559   snat_static_map_resolve_t *rp;
4560   u32 *indices_to_delete = 0;
4561   int i, j;
4562   u32 *auto_add_sw_if_indices =
4563     twice_nat ? sm->
4564     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4565
4566   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4567     );
4568
4569   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4570     {
4571       if (auto_add_sw_if_indices[i] == sw_if_index)
4572         {
4573           if (is_del)
4574             {
4575               /* if have address remove it */
4576               if (first_int_addr)
4577                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4578               else
4579                 {
4580                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4581                     {
4582                       rp = sm->to_resolve + j;
4583                       if (rp->sw_if_index == sw_if_index)
4584                         vec_add1 (indices_to_delete, j);
4585                     }
4586                   if (vec_len (indices_to_delete))
4587                     {
4588                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4589                         vec_del1 (sm->to_resolve, j);
4590                       vec_free (indices_to_delete);
4591                     }
4592                 }
4593               if (twice_nat)
4594                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4595               else
4596                 vec_del1 (sm->auto_add_sw_if_indices, i);
4597             }
4598           else
4599             return VNET_API_ERROR_VALUE_EXIST;
4600
4601           return 0;
4602         }
4603     }
4604
4605   if (is_del)
4606     return VNET_API_ERROR_NO_SUCH_ENTRY;
4607
4608   /* add to the auto-address list */
4609   if (twice_nat)
4610     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4611   else
4612     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4613
4614   /* If the address is already bound - or static - add it now */
4615   if (first_int_addr)
4616     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4617
4618   return 0;
4619 }
4620
4621 int
4622 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4623                    nat_protocol_t proto, u32 vrf_id, int is_in)
4624 {
4625   snat_main_per_thread_data_t *tsm;
4626   clib_bihash_kv_8_8_t kv, value;
4627   ip4_header_t ip;
4628   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4629   snat_session_t *s;
4630   clib_bihash_8_8_t *t;
4631
4632   if (sm->endpoint_dependent)
4633     return VNET_API_ERROR_UNSUPPORTED;
4634
4635   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4636   if (sm->num_workers > 1)
4637     tsm =
4638       vec_elt_at_index (sm->per_thread_data,
4639                         sm->worker_in2out_cb (&ip, fib_index, 0));
4640   else
4641     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4642
4643   init_nat_k (&kv, *addr, port, fib_index, proto);
4644   t = is_in ? &tsm->in2out : &tsm->out2in;
4645   if (!clib_bihash_search_8_8 (t, &kv, &value))
4646     {
4647       if (pool_is_free_index (tsm->sessions, value.value))
4648         return VNET_API_ERROR_UNSPECIFIED;
4649
4650       s = pool_elt_at_index (tsm->sessions, value.value);
4651       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4652       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4653       return 0;
4654     }
4655
4656   return VNET_API_ERROR_NO_SUCH_ENTRY;
4657 }
4658
4659 int
4660 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4661                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4662                       u32 vrf_id, int is_in)
4663 {
4664   ip4_header_t ip;
4665   clib_bihash_16_8_t *t;
4666   clib_bihash_kv_16_8_t kv, value;
4667   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4668   snat_session_t *s;
4669   snat_main_per_thread_data_t *tsm;
4670
4671   if (!sm->endpoint_dependent)
4672     return VNET_API_ERROR_FEATURE_DISABLED;
4673
4674   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4675   if (sm->num_workers > 1)
4676     tsm =
4677       vec_elt_at_index (sm->per_thread_data,
4678                         sm->worker_in2out_cb (&ip, fib_index, 0));
4679   else
4680     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4681
4682   t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
4683   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
4684   if (clib_bihash_search_16_8 (t, &kv, &value))
4685     {
4686       return VNET_API_ERROR_NO_SUCH_ENTRY;
4687     }
4688
4689   if (pool_is_free_index (tsm->sessions, value.value))
4690     return VNET_API_ERROR_UNSPECIFIED;
4691   s = pool_elt_at_index (tsm->sessions, value.value);
4692   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4693   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
4694   return 0;
4695 }
4696
4697 void
4698 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4699 {
4700   snat_main_t *sm = &snat_main;
4701
4702   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4703   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4704   sm->psid = psid;
4705   sm->psid_offset = psid_offset;
4706   sm->psid_length = psid_length;
4707 }
4708
4709 void
4710 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4711 {
4712   snat_main_t *sm = &snat_main;
4713
4714   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4715   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4716   sm->start_port = start_port;
4717   sm->end_port = end_port;
4718 }
4719
4720 void
4721 nat_set_alloc_addr_and_port_default (void)
4722 {
4723   snat_main_t *sm = &snat_main;
4724
4725   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4726   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4727 }
4728
4729 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4730                                  vlib_node_runtime_t * node,
4731                                  vlib_frame_t * frame)
4732 {
4733   return 0;
4734 }
4735
4736 /* *INDENT-OFF* */
4737 VLIB_REGISTER_NODE (nat_default_node) = {
4738   .name = "nat-default",
4739   .vector_size = sizeof (u32),
4740   .format_trace = 0,
4741   .type = VLIB_NODE_TYPE_INTERNAL,
4742   .n_errors = 0,
4743   .n_next_nodes = NAT_N_NEXT,
4744   .next_nodes = {
4745     [NAT_NEXT_DROP] = "error-drop",
4746     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4747     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4748     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4749     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
4750     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4751     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4752     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4753     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4754     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4755   },
4756 };
4757 /* *INDENT-ON* */
4758
4759 /*
4760  * fd.io coding-style-patch-verification: ON
4761  *
4762  * Local Variables:
4763  * eval: (c-set-style "gnu")
4764  * End:
4765  */