src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat64.h>
  26 #include <nat/nat_inlines.h>
  27 #include <nat/nat44/inlines.h>
  28 #include <nat/nat_affinity.h>
  29 #include <nat/nat_syslog.h>
  30 #include <nat/nat_ha.h>
  31 #include <vnet/fib/fib_table.h>
  32 #include <vnet/fib/ip4_fib.h>
  33 #include <vnet/ip/reass/ip4_sv_reass.h>
  34 #include <vppinfra/bihash_16_8.h>
  35 #include <nat/nat44/ed_inlines.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 fib_source_t nat_fib_src_hi;
  42 fib_source_t nat_fib_src_low;
  43
  44 /* *INDENT-OFF* */
  45 /* Hook up input features */
  46 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat-pre-in2out",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-sv-reassembly-feature"),
  51 };
  52 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat-pre-out2in",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  56                                "ip4-dhcp-client-detect",
  57                                "ip4-sv-reassembly-feature"),
  58 };
  59 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-in2out-worker-handoff",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-out2in-worker-handoff",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-in2out",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  74 };
  75 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-out2in",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  79                                "ip4-dhcp-client-detect"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-classify",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  85 };
  86 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-ed-in2out",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  90 };
  91 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-ed-out2in",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  95                                "ip4-dhcp-client-detect"),
  96 };
  97 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-ed-classify",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-handoff-classify",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-in2out-fast",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 111 };
 112 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-out2in-fast",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 116                                "ip4-dhcp-client-detect"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-hairpin-dst",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ed-hairpin-dst",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 127 };
 128
 129 /* Hook up output features */
 130 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 131   .arc_name = "ip4-output",
 132   .node_name = "nat44-in2out-output",
 133   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 136   .arc_name = "ip4-output",
 137   .node_name = "nat44-in2out-output-worker-handoff",
 138   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 141   .arc_name = "ip4-output",
 142   .node_name = "nat44-hairpin-src",
 143   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 144 };
 145 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-ed-in2out-output",
 148   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 149   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 150 };
 151 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-ed-hairpin-src",
 154   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 155   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 156 };
 157
 158 /* Hook up ip4-local features */
 159 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 160 {
 161   .arc_name = "ip4-local",
 162   .node_name = "nat44-hairpinning",
 163   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 164 };
 165 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 166 {
 167   .arc_name = "ip4-local",
 168   .node_name = "nat44-ed-hairpinning",
 169   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 170 };
 171
 172
 173 VLIB_PLUGIN_REGISTER () = {
 174     .version = VPP_BUILD_VER,
 175     .description = "Network Address Translation (NAT)",
 176 };
 177 /* *INDENT-ON* */
 178
 179 void
 180 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 181                        u8 is_ha)
 182 {
 183   clib_bihash_kv_8_8_t kv;
 184   u8 proto;
 185   u16 r_port, l_port;
 186   ip4_address_t *l_addr, *r_addr;
 187   u32 fib_index = 0;
 188   clib_bihash_kv_16_8_t ed_kv;
 189   snat_main_per_thread_data_t *tsm =
 190     vec_elt_at_index (sm->per_thread_data, thread_index);
 191
 192   if (is_ed_session (s))
 193     {
 194       per_vrf_sessions_unregister_session (s, thread_index);
 195     }
 196
 197   if (is_fwd_bypass_session (s))
 198     {
 199       if (snat_is_unk_proto_session (s))
 200         {
 201           init_ed_k (&ed_kv, s->in2out.addr, 0, s->ext_host_addr, 0, 0,
 202                      s->in2out.port);
 203         }
 204       else
 205         {
 206           l_port = s->in2out.port;
 207           r_port = s->ext_host_port;
 208           l_addr = &s->in2out.addr;
 209           r_addr = &s->ext_host_addr;
 210           proto = nat_proto_to_ip_proto (s->nat_proto);
 211           init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index,
 212                      proto);
 213         }
 214       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 215         nat_elog_warn ("in2out_ed key del failed");
 216       return;
 217     }
 218
 219   /* session lookup tables */
 220   if (is_ed_session (s))
 221     {
 222       if (is_affinity_sessions (s))
 223         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 224                              s->nat_proto, s->out2in.port);
 225       l_addr = &s->out2in.addr;
 226       r_addr = &s->ext_host_addr;
 227       fib_index = s->out2in.fib_index;
 228       if (snat_is_unk_proto_session (s))
 229         {
 230           proto = s->in2out.port;
 231           r_port = 0;
 232           l_port = 0;
 233         }
 234       else
 235         {
 236           proto = nat_proto_to_ip_proto (s->nat_proto);
 237           l_port = s->out2in.port;
 238           r_port = s->ext_host_port;
 239         }
 240       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 241       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 242         nat_elog_warn ("out2in_ed key del failed");
 243       l_addr = &s->in2out.addr;
 244       fib_index = s->in2out.fib_index;
 245       if (!snat_is_unk_proto_session (s))
 246         l_port = s->in2out.port;
 247       if (is_twice_nat_session (s))
 248         {
 249           r_addr = &s->ext_host_nat_addr;
 250           r_port = s->ext_host_nat_port;
 251         }
 252       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 253       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 254         nat_elog_warn ("in2out_ed key del failed");
 255
 256       if (!is_ha)
 257         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 258                                &s->in2out.addr, s->in2out.port,
 259                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 260                                &s->out2in.addr, s->out2in.port,
 261                                &s->ext_host_addr, s->ext_host_port,
 262                                s->nat_proto, is_twice_nat_session (s));
 263     }
 264   else
 265     {
 266       init_nat_i2o_k (&kv, s);
 267       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 268         nat_elog_warn ("in2out key del failed");
 269       init_nat_o2i_k (&kv, s);
 270       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 271         nat_elog_warn ("out2in key del failed");
 272
 273       if (!is_ha)
 274         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 275                                  &s->in2out.addr, s->in2out.port,
 276                                  &s->out2in.addr, s->out2in.port,
 277                                  s->nat_proto);
 278     }
 279
 280   if (snat_is_unk_proto_session (s))
 281     return;
 282
 283   if (!is_ha)
 284     {
 285       /* log NAT event */
 286       snat_ipfix_logging_nat44_ses_delete (thread_index,
 287                                            s->in2out.addr.as_u32,
 288                                            s->out2in.addr.as_u32,
 289                                            s->nat_proto,
 290                                            s->in2out.port,
 291                                            s->out2in.port,
 292                                            s->in2out.fib_index);
 293
 294       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 295                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 296                    thread_index);
 297     }
 298
 299   /* Twice NAT address and port for external host */
 300   if (is_twice_nat_session (s))
 301     {
 302       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 303                                           thread_index,
 304                                           &s->ext_host_nat_addr,
 305                                           s->ext_host_nat_port, s->nat_proto);
 306     }
 307
 308   if (snat_is_session_static (s))
 309     return;
 310
 311   snat_free_outside_address_and_port (sm->addresses, thread_index,
 312                                       &s->out2in.addr, s->out2in.port,
 313                                       s->nat_proto);
 314 }
 315
 316 void
 317 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 318                          u32 thread_index, u8 is_ha)
 319 {
 320   u8 proto;
 321   u16 r_port, l_port;
 322   ip4_address_t *l_addr, *r_addr;
 323   u32 fib_index;
 324   clib_bihash_kv_16_8_t ed_kv;
 325   snat_main_per_thread_data_t *tsm =
 326     vec_elt_at_index (sm->per_thread_data, thread_index);
 327
 328   if (is_fwd_bypass_session (s))
 329     {
 330       if (snat_is_unk_proto_session (s))
 331         {
 332           proto = s->in2out.port;
 333           r_port = 0;
 334           l_port = 0;
 335         }
 336       else
 337         {
 338           proto = nat_proto_to_ip_proto (s->nat_proto);
 339           l_port = s->in2out.port;
 340           r_port = s->ext_host_port;
 341         }
 342
 343       l_addr = &s->in2out.addr;
 344       r_addr = &s->ext_host_addr;
 345       fib_index = 0;
 346       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 347
 348       if (PREDICT_FALSE
 349           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 350         nat_elog_warn ("in2out_ed key del failed");
 351       return;
 352     }
 353
 354   /* session lookup tables */
 355   if (is_affinity_sessions (s))
 356     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 357                          s->nat_proto, s->out2in.port);
 358   l_addr = &s->out2in.addr;
 359   r_addr = &s->ext_host_addr;
 360   fib_index = s->out2in.fib_index;
 361   if (snat_is_unk_proto_session (s))
 362     {
 363       proto = s->in2out.port;
 364       r_port = 0;
 365       l_port = 0;
 366     }
 367   else
 368     {
 369       proto = nat_proto_to_ip_proto (s->nat_proto);
 370       l_port = s->out2in.port;
 371       r_port = s->ext_host_port;
 372     }
 373   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 374
 375   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0)))
 376     nat_elog_warn ("out2in_ed key del failed");
 377
 378   l_addr = &s->in2out.addr;
 379   fib_index = s->in2out.fib_index;
 380
 381   if (!snat_is_unk_proto_session (s))
 382     l_port = s->in2out.port;
 383
 384   if (is_twice_nat_session (s))
 385     {
 386       r_addr = &s->ext_host_nat_addr;
 387       r_port = s->ext_host_nat_port;
 388     }
 389   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 390
 391   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 392     nat_elog_warn ("in2out_ed key del failed");
 393
 394   if (!is_ha)
 395     {
 396       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 397                              &s->in2out.addr, s->in2out.port,
 398                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 399                              &s->out2in.addr, s->out2in.port,
 400                              &s->ext_host_addr, s->ext_host_port,
 401                              s->nat_proto, is_twice_nat_session (s));
 402     }
 403
 404   if (snat_is_unk_proto_session (s))
 405     return;
 406
 407   if (!is_ha)
 408     {
 409       snat_ipfix_logging_nat44_ses_delete (thread_index,
 410                                            s->in2out.addr.as_u32,
 411                                            s->out2in.addr.as_u32,
 412                                            s->nat_proto,
 413                                            s->in2out.port,
 414                                            s->out2in.port,
 415                                            s->in2out.fib_index);
 416       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 417                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 418                    thread_index);
 419     }
 420
 421   /* Twice NAT address and port for external host */
 422   if (is_twice_nat_session (s))
 423     {
 424       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 425                                           thread_index,
 426                                           &s->ext_host_nat_addr,
 427                                           s->ext_host_nat_port, s->nat_proto);
 428     }
 429
 430   if (snat_is_session_static (s))
 431     return;
 432
 433   snat_free_outside_address_and_port (sm->addresses, thread_index,
 434                                       &s->out2in.addr, s->out2in.port,
 435                                       s->nat_proto);
 436 }
 437
 438
 439 snat_user_t *
 440 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 441                         u32 thread_index)
 442 {
 443   snat_user_t *u = 0;
 444   snat_user_key_t user_key;
 445   clib_bihash_kv_8_8_t kv, value;
 446   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 447   dlist_elt_t *per_user_list_head_elt;
 448
 449   user_key.addr.as_u32 = addr->as_u32;
 450   user_key.fib_index = fib_index;
 451   kv.key = user_key.as_u64;
 452
 453   /* Ever heard of the "user" = src ip4 address before? */
 454   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 455     {
 456       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 457         {
 458           vlib_increment_simple_counter (&sm->user_limit_reached,
 459                                          thread_index, 0, 1);
 460           nat_elog_warn ("maximum user limit reached");
 461           return NULL;
 462         }
 463       /* no, make a new one */
 464       pool_get (tsm->users, u);
 465       clib_memset (u, 0, sizeof (*u));
 466
 467       u->addr.as_u32 = addr->as_u32;
 468       u->fib_index = fib_index;
 469
 470       pool_get (tsm->list_pool, per_user_list_head_elt);
 471
 472       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 473         tsm->list_pool;
 474
 475       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 476
 477       kv.value = u - tsm->users;
 478
 479       /* add user */
 480       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 481         {
 482           nat_elog_warn ("user_hash key add failed");
 483           nat44_delete_user_with_no_session (sm, u, thread_index);
 484           return NULL;
 485         }
 486
 487       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 488                                pool_elts (tsm->users));
 489     }
 490   else
 491     {
 492       u = pool_elt_at_index (tsm->users, value.value);
 493     }
 494
 495   return u;
 496 }
 497
 498 snat_session_t *
 499 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 500                               u32 thread_index, f64 now)
 501 {
 502   snat_session_t *s;
 503   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 504   u32 oldest_per_user_translation_list_index, session_index;
 505   dlist_elt_t *oldest_per_user_translation_list_elt;
 506   dlist_elt_t *per_user_translation_list_elt;
 507
 508   /* Over quota? Recycle the least recently used translation */
 509   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 510     {
 511       oldest_per_user_translation_list_index =
 512         clib_dlist_remove_head (tsm->list_pool,
 513                                 u->sessions_per_user_list_head_index);
 514
 515       ASSERT (oldest_per_user_translation_list_index != ~0);
 516
 517       /* Add it back to the end of the LRU list */
 518       clib_dlist_addtail (tsm->list_pool,
 519                           u->sessions_per_user_list_head_index,
 520                           oldest_per_user_translation_list_index);
 521       /* Get the list element */
 522       oldest_per_user_translation_list_elt =
 523         pool_elt_at_index (tsm->list_pool,
 524                            oldest_per_user_translation_list_index);
 525
 526       /* Get the session index from the list element */
 527       session_index = oldest_per_user_translation_list_elt->value;
 528
 529       /* Get the session */
 530       s = pool_elt_at_index (tsm->sessions, session_index);
 531       nat_free_session_data (sm, s, thread_index, 0);
 532       if (snat_is_session_static (s))
 533         u->nstaticsessions--;
 534       else
 535         u->nsessions--;
 536       s->flags = 0;
 537       s->total_bytes = 0;
 538       s->total_pkts = 0;
 539       s->state = 0;
 540       s->ext_host_addr.as_u32 = 0;
 541       s->ext_host_port = 0;
 542       s->ext_host_nat_addr.as_u32 = 0;
 543       s->ext_host_nat_port = 0;
 544     }
 545   else
 546     {
 547       pool_get (tsm->sessions, s);
 548       clib_memset (s, 0, sizeof (*s));
 549
 550       /* Create list elts */
 551       pool_get (tsm->list_pool, per_user_translation_list_elt);
 552       clib_dlist_init (tsm->list_pool,
 553                        per_user_translation_list_elt - tsm->list_pool);
 554
 555       per_user_translation_list_elt->value = s - tsm->sessions;
 556       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 557       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 558
 559       clib_dlist_addtail (tsm->list_pool,
 560                           s->per_user_list_head_index,
 561                           per_user_translation_list_elt - tsm->list_pool);
 562
 563       s->user_index = u - tsm->users;
 564       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 565                                pool_elts (tsm->sessions));
 566     }
 567
 568   s->ha_last_refreshed = now;
 569
 570   return s;
 571 }
 572
 573 void
 574 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 575                           int is_add)
 576 {
 577   fib_prefix_t prefix = {
 578     .fp_len = p_len,
 579     .fp_proto = FIB_PROTOCOL_IP4,
 580     .fp_addr = {
 581                 .ip4.as_u32 = addr->as_u32,
 582                 },
 583   };
 584   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 585
 586   if (is_add)
 587     fib_table_entry_update_one_path (fib_index,
 588                                      &prefix,
 589                                      nat_fib_src_low,
 590                                      (FIB_ENTRY_FLAG_CONNECTED |
 591                                       FIB_ENTRY_FLAG_LOCAL |
 592                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 593                                      DPO_PROTO_IP4,
 594                                      NULL,
 595                                      sw_if_index,
 596                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 597   else
 598     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 599 }
 600
 601 int
 602 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 603                   u8 twice_nat)
 604 {
 605   snat_address_t *ap;
 606   snat_interface_t *i;
 607   vlib_thread_main_t *tm = vlib_get_thread_main ();
 608
 609   if (twice_nat && !sm->endpoint_dependent)
 610     return VNET_API_ERROR_FEATURE_DISABLED;
 611
 612   /* Check if address already exists */
 613   /* *INDENT-OFF* */
 614   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 615     {
 616       if (ap->addr.as_u32 == addr->as_u32)
 617         return VNET_API_ERROR_VALUE_EXIST;
 618     }
 619   /* *INDENT-ON* */
 620
 621   if (twice_nat)
 622     vec_add2 (sm->twice_nat_addresses, ap, 1);
 623   else
 624     vec_add2 (sm->addresses, ap, 1);
 625
 626   ap->addr = *addr;
 627   if (vrf_id != ~0)
 628     ap->fib_index =
 629       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 630                                          nat_fib_src_low);
 631   else
 632     ap->fib_index = ~0;
 633 #define _(N, i, n, s) \
 634   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 635   ap->busy_##n##_ports = 0; \
 636   ap->busy_##n##_ports_per_thread = 0;\
 637   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 638   foreach_nat_protocol
 639 #undef _
 640     if (twice_nat)
 641     return 0;
 642
 643   /* Add external address to FIB */
 644   /* *INDENT-OFF* */
 645   pool_foreach (i, sm->interfaces,
 646   ({
 647     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 648       continue;
 649
 650     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 651     break;
 652   }));
 653   pool_foreach (i, sm->output_feature_interfaces,
 654   ({
 655     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 656       continue;
 657
 658     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 659     break;
 660   }));
 661   /* *INDENT-ON* */
 662
 663   return 0;
 664 }
 665
 666 static int
 667 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 668 {
 669   snat_static_mapping_t *m;
 670   /* *INDENT-OFF* */
 671   pool_foreach (m, sm->static_mappings,
 672   ({
 673       if (is_addr_only_static_mapping (m) ||
 674           is_out2in_only_static_mapping (m) ||
 675           is_identity_static_mapping (m))
 676         continue;
 677       if (m->external_addr.as_u32 == addr.as_u32)
 678         return 1;
 679   }));
 680   /* *INDENT-ON* */
 681
 682   return 0;
 683 }
 684
 685 static void
 686 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 687                                        ip4_address_t l_addr,
 688                                        u16 l_port,
 689                                        u32 sw_if_index,
 690                                        u16 e_port,
 691                                        u32 vrf_id,
 692                                        nat_protocol_t proto,
 693                                        int addr_only, int is_add, u8 * tag,
 694                                        int twice_nat, int out2in_only,
 695                                        int identity_nat,
 696                                        ip4_address_t pool_addr, int exact)
 697 {
 698   snat_static_map_resolve_t *rp;
 699
 700   vec_add2 (sm->to_resolve, rp, 1);
 701   rp->l_addr.as_u32 = l_addr.as_u32;
 702   rp->l_port = l_port;
 703   rp->sw_if_index = sw_if_index;
 704   rp->e_port = e_port;
 705   rp->vrf_id = vrf_id;
 706   rp->proto = proto;
 707   rp->addr_only = addr_only;
 708   rp->is_add = is_add;
 709   rp->twice_nat = twice_nat;
 710   rp->out2in_only = out2in_only;
 711   rp->identity_nat = identity_nat;
 712   rp->tag = vec_dup (tag);
 713   rp->pool_addr = pool_addr;
 714   rp->exact = exact;
 715 }
 716
 717 static u32
 718 get_thread_idx_by_port (u16 e_port)
 719 {
 720   snat_main_t *sm = &snat_main;
 721   u32 thread_idx = sm->num_workers;
 722   if (sm->num_workers > 1)
 723     {
 724       thread_idx =
 725         sm->first_worker_index +
 726         sm->workers[(e_port - 1024) / sm->port_per_thread];
 727     }
 728   return thread_idx;
 729 }
 730
 731 void
 732 snat_static_mapping_del_sessions (snat_main_t * sm,
 733                                   snat_main_per_thread_data_t * tsm,
 734                                   snat_user_key_t u_key, int addr_only,
 735                                   ip4_address_t e_addr, u16 e_port)
 736 {
 737   clib_bihash_kv_8_8_t kv, value;
 738   kv.key = u_key.as_u64;
 739   u64 user_index;
 740   dlist_elt_t *head, *elt;
 741   snat_user_t *u;
 742   snat_session_t *s;
 743   u32 elt_index, head_index, ses_index;
 744   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 745     {
 746       user_index = value.value;
 747       u = pool_elt_at_index (tsm->users, user_index);
 748       if (u->nstaticsessions)
 749         {
 750           head_index = u->sessions_per_user_list_head_index;
 751           head = pool_elt_at_index (tsm->list_pool, head_index);
 752           elt_index = head->next;
 753           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 754           ses_index = elt->value;
 755           while (ses_index != ~0)
 756             {
 757               s = pool_elt_at_index (tsm->sessions, ses_index);
 758               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 759               ses_index = elt->value;
 760
 761               if (!addr_only)
 762                 {
 763                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 764                       (s->out2in.port != e_port))
 765                     continue;
 766                 }
 767
 768               if (is_lb_session (s))
 769                 continue;
 770
 771               if (!snat_is_session_static (s))
 772                 continue;
 773
 774               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 775               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 776
 777               if (!addr_only)
 778                 break;
 779             }
 780         }
 781     }
 782 }
 783
 784 void
 785 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 786                                      snat_main_per_thread_data_t * tsm,
 787                                      ip4_address_t l_addr,
 788                                      u16 l_port,
 789                                      u8 protocol,
 790                                      u32 fib_index, int addr_only,
 791                                      ip4_address_t e_addr, u16 e_port)
 792 {
 793   snat_session_t *s;
 794   u32 *indexes_to_free = NULL;
 795   /* *INDENT-OFF* */
 796   pool_foreach (s, tsm->sessions, {
 797     if (s->in2out.fib_index != fib_index ||
 798         s->in2out.addr.as_u32 != l_addr.as_u32)
 799       {
 800         continue;
 801       }
 802     if (!addr_only)
 803       {
 804         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 805             s->out2in.port != e_port ||
 806             s->in2out.port != l_port ||
 807             s->nat_proto != protocol)
 808           continue;
 809       }
 810
 811     if (is_lb_session (s))
 812       continue;
 813     if (!snat_is_session_static (s))
 814       continue;
 815     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 816     vec_add1 (indexes_to_free, s - tsm->sessions);
 817     if (!addr_only)
 818       break;
 819   });
 820   /* *INDENT-ON* */
 821   u32 *ses_index;
 822   vec_foreach (ses_index, indexes_to_free)
 823   {
 824     s = pool_elt_at_index (tsm->sessions, *ses_index);
 825     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 826   }
 827   vec_free (indexes_to_free);
 828 }
 829
 830 int
 831 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 832                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 833                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 834                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 835                          u8 identity_nat, ip4_address_t pool_addr, int exact)
 836 {
 837   snat_main_t *sm = &snat_main;
 838   snat_static_mapping_t *m;
 839   clib_bihash_kv_8_8_t kv, value;
 840   snat_address_t *a = 0;
 841   u32 fib_index = ~0;
 842   snat_interface_t *interface;
 843   int i;
 844   snat_main_per_thread_data_t *tsm;
 845   snat_user_key_t u_key;
 846   snat_user_t *u;
 847   dlist_elt_t *head, *elt;
 848   u32 elt_index, head_index;
 849   u32 ses_index;
 850   u64 user_index;
 851   snat_session_t *s;
 852   snat_static_map_resolve_t *rp, *rp_match = 0;
 853   nat44_lb_addr_port_t *local;
 854   u32 find = ~0;
 855
 856   if (!sm->endpoint_dependent)
 857     {
 858       if (twice_nat || out2in_only)
 859         return VNET_API_ERROR_FEATURE_DISABLED;
 860     }
 861
 862   /* If the external address is a specific interface address */
 863   if (sw_if_index != ~0)
 864     {
 865       ip4_address_t *first_int_addr;
 866
 867       for (i = 0; i < vec_len (sm->to_resolve); i++)
 868         {
 869           rp = sm->to_resolve + i;
 870           if (rp->sw_if_index != sw_if_index ||
 871               rp->l_addr.as_u32 != l_addr.as_u32 ||
 872               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 873             continue;
 874
 875           if (!addr_only)
 876             {
 877               if ((rp->l_port != l_port && rp->e_port != e_port)
 878                   || rp->proto != proto)
 879                 continue;
 880             }
 881
 882           rp_match = rp;
 883           break;
 884         }
 885
 886       /* Might be already set... */
 887       first_int_addr = ip4_interface_first_address
 888         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 889
 890       if (is_add)
 891         {
 892           if (rp_match)
 893             return VNET_API_ERROR_VALUE_EXIST;
 894
 895           snat_add_static_mapping_when_resolved
 896             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 897              addr_only, is_add, tag, twice_nat, out2in_only,
 898              identity_nat, pool_addr, exact);
 899
 900           /* DHCP resolution required? */
 901           if (first_int_addr == 0)
 902             {
 903               return 0;
 904             }
 905           else
 906             {
 907               e_addr.as_u32 = first_int_addr->as_u32;
 908               /* Identity mapping? */
 909               if (l_addr.as_u32 == 0)
 910                 l_addr.as_u32 = e_addr.as_u32;
 911             }
 912         }
 913       else
 914         {
 915           if (!rp_match)
 916             return VNET_API_ERROR_NO_SUCH_ENTRY;
 917
 918           vec_del1 (sm->to_resolve, i);
 919
 920           if (first_int_addr)
 921             {
 922               e_addr.as_u32 = first_int_addr->as_u32;
 923               /* Identity mapping? */
 924               if (l_addr.as_u32 == 0)
 925                 l_addr.as_u32 = e_addr.as_u32;
 926             }
 927           else
 928             return 0;
 929         }
 930     }
 931
 932   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 933   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 934     m = 0;
 935   else
 936     m = pool_elt_at_index (sm->static_mappings, value.value);
 937
 938   if (is_add)
 939     {
 940       if (m)
 941         {
 942           if (is_identity_static_mapping (m))
 943             {
 944               /* *INDENT-OFF* */
 945               pool_foreach (local, m->locals,
 946               ({
 947                 if (local->vrf_id == vrf_id)
 948                   return VNET_API_ERROR_VALUE_EXIST;
 949               }));
 950               /* *INDENT-ON* */
 951               pool_get (m->locals, local);
 952               local->vrf_id = vrf_id;
 953               local->fib_index =
 954                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 955                                                    nat_fib_src_low);
 956               init_nat_kv (&kv, m->local_addr, m->local_port,
 957                            local->fib_index, m->proto,
 958                            m - sm->static_mappings);
 959               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 960               return 0;
 961             }
 962           else
 963             return VNET_API_ERROR_VALUE_EXIST;
 964         }
 965
 966       if (twice_nat && addr_only)
 967         return VNET_API_ERROR_UNSUPPORTED;
 968
 969       /* Convert VRF id to FIB index */
 970       if (vrf_id != ~0)
 971         fib_index =
 972           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 973                                              nat_fib_src_low);
 974       /* If not specified use inside VRF id from SNAT plugin startup config */
 975       else
 976         {
 977           fib_index = sm->inside_fib_index;
 978           vrf_id = sm->inside_vrf_id;
 979           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 980         }
 981
 982       if (!(out2in_only || identity_nat))
 983         {
 984           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
 985                       addr_only ? 0 : proto);
 986           if (!clib_bihash_search_8_8
 987               (&sm->static_mapping_by_local, &kv, &value))
 988             return VNET_API_ERROR_VALUE_EXIST;
 989         }
 990
 991       /* Find external address in allocated addresses and reserve port for
 992          address and port pair mapping when dynamic translations enabled */
 993       if (!(addr_only || sm->static_mapping_only || out2in_only))
 994         {
 995           for (i = 0; i < vec_len (sm->addresses); i++)
 996             {
 997               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 998                 {
 999                   a = sm->addresses + i;
1000                   /* External port must be unused */
1001                   switch (proto)
1002                     {
1003 #define _(N, j, n, s) \
1004                     case NAT_PROTOCOL_##N: \
1005                       if (a->busy_##n##_port_refcounts[e_port]) \
1006                         return VNET_API_ERROR_INVALID_VALUE; \
1007                       ++a->busy_##n##_port_refcounts[e_port]; \
1008                       if (e_port > 1024) \
1009                         { \
1010                           a->busy_##n##_ports++; \
1011                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1012                         } \
1013                       break;
1014                       foreach_nat_protocol
1015 #undef _
1016                     default:
1017                       nat_elog_info ("unknown protocol");
1018                       return VNET_API_ERROR_INVALID_VALUE_2;
1019                     }
1020                   break;
1021                 }
1022             }
1023           /* External address must be allocated */
1024           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1025             {
1026               if (sw_if_index != ~0)
1027                 {
1028                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1029                     {
1030                       rp = sm->to_resolve + i;
1031                       if (rp->addr_only)
1032                         continue;
1033                       if (rp->sw_if_index != sw_if_index &&
1034                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1035                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1036                           rp->e_port != e_port && rp->proto != proto)
1037                         continue;
1038
1039                       vec_del1 (sm->to_resolve, i);
1040                       break;
1041                     }
1042                 }
1043               return VNET_API_ERROR_NO_SUCH_ENTRY;
1044             }
1045         }
1046
1047       pool_get (sm->static_mappings, m);
1048       clib_memset (m, 0, sizeof (*m));
1049       m->tag = vec_dup (tag);
1050       m->local_addr = l_addr;
1051       m->external_addr = e_addr;
1052       m->twice_nat = twice_nat;
1053
1054       if (twice_nat == TWICE_NAT && exact)
1055         {
1056           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
1057           m->pool_addr = pool_addr;
1058         }
1059
1060       if (out2in_only)
1061         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1062       if (addr_only)
1063         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1064       if (identity_nat)
1065         {
1066           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1067           pool_get (m->locals, local);
1068           local->vrf_id = vrf_id;
1069           local->fib_index = fib_index;
1070         }
1071       else
1072         {
1073           m->vrf_id = vrf_id;
1074           m->fib_index = fib_index;
1075         }
1076       if (!addr_only)
1077         {
1078           m->local_port = l_port;
1079           m->external_port = e_port;
1080           m->proto = proto;
1081         }
1082
1083       if (sm->num_workers > 1)
1084         {
1085           ip4_header_t ip = {
1086             .src_address = m->local_addr,
1087           };
1088           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1089           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1090         }
1091       else
1092         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1093
1094       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
1095                    m - sm->static_mappings);
1096       if (!out2in_only)
1097         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1098
1099       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1100                    m - sm->static_mappings);
1101       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1102
1103       /* Delete dynamic sessions matching local address (+ local port) */
1104       if (!(sm->static_mapping_only))
1105         {
1106           u_key.addr = m->local_addr;
1107           u_key.fib_index = m->fib_index;
1108           kv.key = u_key.as_u64;
1109           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1110             {
1111               user_index = value.value;
1112               u = pool_elt_at_index (tsm->users, user_index);
1113               if (u->nsessions)
1114                 {
1115                   head_index = u->sessions_per_user_list_head_index;
1116                   head = pool_elt_at_index (tsm->list_pool, head_index);
1117                   elt_index = head->next;
1118                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1119                   ses_index = elt->value;
1120                   while (ses_index != ~0)
1121                     {
1122                       s = pool_elt_at_index (tsm->sessions, ses_index);
1123                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1124                       ses_index = elt->value;
1125
1126                       if (snat_is_session_static (s))
1127                         continue;
1128
1129                       if (!addr_only && s->in2out.port != m->local_port)
1130                         continue;
1131
1132                       nat_free_session_data (sm, s,
1133                                              tsm - sm->per_thread_data, 0);
1134                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1135
1136                       if (!addr_only && !sm->endpoint_dependent)
1137                         break;
1138                     }
1139                 }
1140             }
1141         }
1142     }
1143   else
1144     {
1145       if (!m)
1146         {
1147           if (sw_if_index != ~0)
1148             return 0;
1149           else
1150             return VNET_API_ERROR_NO_SUCH_ENTRY;
1151         }
1152
1153       if (identity_nat)
1154         {
1155           if (vrf_id == ~0)
1156             vrf_id = sm->inside_vrf_id;
1157
1158           /* *INDENT-OFF* */
1159           pool_foreach (local, m->locals,
1160           ({
1161             if (local->vrf_id == vrf_id)
1162               find = local - m->locals;
1163           }));
1164           /* *INDENT-ON* */
1165           if (find == ~0)
1166             return VNET_API_ERROR_NO_SUCH_ENTRY;
1167
1168           local = pool_elt_at_index (m->locals, find);
1169           fib_index = local->fib_index;
1170           pool_put (m->locals, local);
1171         }
1172       else
1173         fib_index = m->fib_index;
1174
1175       /* Free external address port */
1176       if (!(addr_only || sm->static_mapping_only || out2in_only))
1177         {
1178           for (i = 0; i < vec_len (sm->addresses); i++)
1179             {
1180               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1181                 {
1182                   a = sm->addresses + i;
1183                   switch (proto)
1184                     {
1185 #define _(N, j, n, s) \
1186                     case NAT_PROTOCOL_##N: \
1187                       --a->busy_##n##_port_refcounts[e_port]; \
1188                       if (e_port > 1024) \
1189                         { \
1190                           a->busy_##n##_ports--; \
1191                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1192                         } \
1193                       break;
1194                       foreach_nat_protocol
1195 #undef _
1196                     default:
1197                       nat_elog_info ("unknown protocol");
1198                       return VNET_API_ERROR_INVALID_VALUE_2;
1199                     }
1200                   break;
1201                 }
1202             }
1203         }
1204
1205       if (sm->num_workers > 1)
1206         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1207       else
1208         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1209
1210       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1211       if (!out2in_only)
1212         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1213
1214       /* Delete session(s) for static mapping if exist */
1215       if (!(sm->static_mapping_only) ||
1216           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1217         {
1218           if (sm->endpoint_dependent)
1219             {
1220               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1221                                                    m->local_port, m->proto,
1222                                                    fib_index, addr_only,
1223                                                    e_addr, e_port);
1224             }
1225           else
1226             {
1227               u_key.addr = m->local_addr;
1228               u_key.fib_index = fib_index;
1229               kv.key = u_key.as_u64;
1230               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1231                                                 e_addr, e_port);
1232             }
1233         }
1234
1235       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1236       if (pool_elts (m->locals))
1237         return 0;
1238
1239       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1240       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1241
1242       vec_free (m->tag);
1243       vec_free (m->workers);
1244       /* Delete static mapping from pool */
1245       pool_put (sm->static_mappings, m);
1246     }
1247
1248   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1249     return 0;
1250
1251   /* Add/delete external address to FIB */
1252   /* *INDENT-OFF* */
1253   pool_foreach (interface, sm->interfaces,
1254   ({
1255     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1256       continue;
1257
1258     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1259     break;
1260   }));
1261   pool_foreach (interface, sm->output_feature_interfaces,
1262   ({
1263     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1264       continue;
1265
1266     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1267     break;
1268   }));
1269   /* *INDENT-ON* */
1270
1271   return 0;
1272 }
1273
1274 int
1275 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1276                                  nat_protocol_t proto,
1277                                  nat44_lb_addr_port_t * locals, u8 is_add,
1278                                  twice_nat_type_t twice_nat, u8 out2in_only,
1279                                  u8 * tag, u32 affinity)
1280 {
1281   snat_main_t *sm = &snat_main;
1282   snat_static_mapping_t *m;
1283   clib_bihash_kv_8_8_t kv, value;
1284   snat_address_t *a = 0;
1285   int i;
1286   nat44_lb_addr_port_t *local;
1287   snat_main_per_thread_data_t *tsm;
1288   snat_session_t *s;
1289   uword *bitmap = 0;
1290
1291   if (!sm->endpoint_dependent)
1292     return VNET_API_ERROR_FEATURE_DISABLED;
1293
1294   init_nat_k (&kv, e_addr, e_port, 0, proto);
1295   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1296     m = 0;
1297   else
1298     m = pool_elt_at_index (sm->static_mappings, value.value);
1299
1300   if (is_add)
1301     {
1302       if (m)
1303         return VNET_API_ERROR_VALUE_EXIST;
1304
1305       if (vec_len (locals) < 2)
1306         return VNET_API_ERROR_INVALID_VALUE;
1307
1308       /* Find external address in allocated addresses and reserve port for
1309          address and port pair mapping when dynamic translations enabled */
1310       if (!(sm->static_mapping_only || out2in_only))
1311         {
1312           for (i = 0; i < vec_len (sm->addresses); i++)
1313             {
1314               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1315                 {
1316                   a = sm->addresses + i;
1317                   /* External port must be unused */
1318                   switch (proto)
1319                     {
1320 #define _(N, j, n, s) \
1321                     case NAT_PROTOCOL_##N: \
1322                       if (a->busy_##n##_port_refcounts[e_port]) \
1323                         return VNET_API_ERROR_INVALID_VALUE; \
1324                       ++a->busy_##n##_port_refcounts[e_port]; \
1325                       if (e_port > 1024) \
1326                         { \
1327                           a->busy_##n##_ports++; \
1328                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1329                         } \
1330                       break;
1331                       foreach_nat_protocol
1332 #undef _
1333                     default:
1334                       nat_elog_info ("unknown protocol");
1335                       return VNET_API_ERROR_INVALID_VALUE_2;
1336                     }
1337                   break;
1338                 }
1339             }
1340           /* External address must be allocated */
1341           if (!a)
1342             return VNET_API_ERROR_NO_SUCH_ENTRY;
1343         }
1344
1345       pool_get (sm->static_mappings, m);
1346       clib_memset (m, 0, sizeof (*m));
1347       m->tag = vec_dup (tag);
1348       m->external_addr = e_addr;
1349       m->external_port = e_port;
1350       m->proto = proto;
1351       m->twice_nat = twice_nat;
1352       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1353       if (out2in_only)
1354         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1355       m->affinity = affinity;
1356
1357       if (affinity)
1358         m->affinity_per_service_list_head_index =
1359           nat_affinity_get_per_service_list_head_index ();
1360       else
1361         m->affinity_per_service_list_head_index = ~0;
1362
1363       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1364                    m - sm->static_mappings);
1365       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1366         {
1367           nat_elog_err ("static_mapping_by_external key add failed");
1368           return VNET_API_ERROR_UNSPECIFIED;
1369         }
1370
1371       for (i = 0; i < vec_len (locals); i++)
1372         {
1373           locals[i].fib_index =
1374             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1375                                                locals[i].vrf_id,
1376                                                nat_fib_src_low);
1377           if (!out2in_only)
1378             {
1379               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1380                            locals[i].fib_index, m->proto,
1381                            m - sm->static_mappings);
1382               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1383             }
1384           locals[i].prefix = (i == 0) ? locals[i].probability :
1385             (locals[i - 1].prefix + locals[i].probability);
1386           pool_get (m->locals, local);
1387           *local = locals[i];
1388           if (sm->num_workers > 1)
1389             {
1390               ip4_header_t ip = {
1391                 .src_address = locals[i].addr,
1392               };
1393               bitmap =
1394                 clib_bitmap_set (bitmap,
1395                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1396                                  1);
1397             }
1398         }
1399
1400       /* Assign workers */
1401       if (sm->num_workers > 1)
1402         {
1403           /* *INDENT-OFF* */
1404           clib_bitmap_foreach (i, bitmap,
1405             ({
1406                vec_add1(m->workers, i);
1407             }));
1408           /* *INDENT-ON* */
1409         }
1410     }
1411   else
1412     {
1413       if (!m)
1414         return VNET_API_ERROR_NO_SUCH_ENTRY;
1415
1416       if (!is_lb_static_mapping (m))
1417         return VNET_API_ERROR_INVALID_VALUE;
1418
1419       /* Free external address port */
1420       if (!(sm->static_mapping_only || out2in_only))
1421         {
1422           for (i = 0; i < vec_len (sm->addresses); i++)
1423             {
1424               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1425                 {
1426                   a = sm->addresses + i;
1427                   switch (proto)
1428                     {
1429 #define _(N, j, n, s) \
1430                     case NAT_PROTOCOL_##N: \
1431                       --a->busy_##n##_port_refcounts[e_port]; \
1432                       if (e_port > 1024) \
1433                         { \
1434                           a->busy_##n##_ports--; \
1435                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1436                         } \
1437                       break;
1438                       foreach_nat_protocol
1439 #undef _
1440                     default:
1441                       nat_elog_info ("unknown protocol");
1442                       return VNET_API_ERROR_INVALID_VALUE_2;
1443                     }
1444                   break;
1445                 }
1446             }
1447         }
1448
1449       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1450       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1451         {
1452           nat_elog_err ("static_mapping_by_external key del failed");
1453           return VNET_API_ERROR_UNSPECIFIED;
1454         }
1455
1456       /* *INDENT-OFF* */
1457       pool_foreach (local, m->locals,
1458       ({
1459           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1460                             nat_fib_src_low);
1461           if (!out2in_only)
1462             {
1463 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1464               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1465                 {
1466                   nat_elog_err ("static_mapping_by_local key del failed");
1467                   return VNET_API_ERROR_UNSPECIFIED;
1468                 }
1469             }
1470
1471           if (sm->num_workers > 1)
1472             {
1473               ip4_header_t ip = {
1474                 .src_address = local->addr,
1475               };
1476               tsm = vec_elt_at_index (sm->per_thread_data,
1477                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1478             }
1479           else
1480             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1481
1482           /* Delete sessions */
1483           pool_foreach (s, tsm->sessions, {
1484             if (!(is_lb_session (s)))
1485               continue;
1486
1487             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1488                 s->in2out.port != local->port)
1489               continue;
1490
1491             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1492             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1493           });
1494       }));
1495       /* *INDENT-ON* */
1496       if (m->affinity)
1497         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1498       pool_free (m->locals);
1499       vec_free (m->tag);
1500       vec_free (m->workers);
1501
1502       pool_put (sm->static_mappings, m);
1503     }
1504
1505   return 0;
1506 }
1507
1508 int
1509 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1510                                        ip4_address_t l_addr, u16 l_port,
1511                                        nat_protocol_t proto, u32 vrf_id,
1512                                        u8 probability, u8 is_add)
1513 {
1514   snat_main_t *sm = &snat_main;
1515   snat_static_mapping_t *m = 0;
1516   clib_bihash_kv_8_8_t kv, value;
1517   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1518   snat_main_per_thread_data_t *tsm;
1519   snat_session_t *s;
1520   u32 *locals = 0;
1521   uword *bitmap = 0;
1522   int i;
1523
1524   if (!sm->endpoint_dependent)
1525     return VNET_API_ERROR_FEATURE_DISABLED;
1526
1527   init_nat_k (&kv, e_addr, e_port, 0, proto);
1528   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1529     m = pool_elt_at_index (sm->static_mappings, value.value);
1530
1531   if (!m)
1532     return VNET_API_ERROR_NO_SUCH_ENTRY;
1533
1534   if (!is_lb_static_mapping (m))
1535     return VNET_API_ERROR_INVALID_VALUE;
1536
1537   /* *INDENT-OFF* */
1538   pool_foreach (local, m->locals,
1539   ({
1540     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1541         (local->vrf_id == vrf_id))
1542       {
1543         match_local = local;
1544         break;
1545       }
1546   }));
1547   /* *INDENT-ON* */
1548
1549   if (is_add)
1550     {
1551       if (match_local)
1552         return VNET_API_ERROR_VALUE_EXIST;
1553
1554       pool_get (m->locals, local);
1555       clib_memset (local, 0, sizeof (*local));
1556       local->addr.as_u32 = l_addr.as_u32;
1557       local->port = l_port;
1558       local->probability = probability;
1559       local->vrf_id = vrf_id;
1560       local->fib_index =
1561         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1562                                            nat_fib_src_low);
1563
1564       if (!is_out2in_only_static_mapping (m))
1565         {
1566           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto,
1567                        m - sm->static_mappings);
1568           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1569             nat_elog_err ("static_mapping_by_local key add failed");
1570         }
1571     }
1572   else
1573     {
1574       if (!match_local)
1575         return VNET_API_ERROR_NO_SUCH_ENTRY;
1576
1577       if (pool_elts (m->locals) < 3)
1578         return VNET_API_ERROR_UNSPECIFIED;
1579
1580       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1581                         nat_fib_src_low);
1582
1583       if (!is_out2in_only_static_mapping (m))
1584         {
1585           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1586           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1587             nat_elog_err ("static_mapping_by_local key del failed");
1588         }
1589
1590       if (sm->num_workers > 1)
1591         {
1592           ip4_header_t ip = {
1593             .src_address = local->addr,
1594           };
1595           tsm = vec_elt_at_index (sm->per_thread_data,
1596                                   sm->worker_in2out_cb (&ip, m->fib_index,
1597                                                         0));
1598         }
1599       else
1600         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1601
1602       /* Delete sessions */
1603       /* *INDENT-OFF* */
1604       pool_foreach (s, tsm->sessions, {
1605         if (!(is_lb_session (s)))
1606           continue;
1607
1608         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1609             s->in2out.port != match_local->port)
1610           continue;
1611
1612         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1613         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1614       });
1615       /* *INDENT-ON* */
1616
1617       pool_put (m->locals, match_local);
1618     }
1619
1620   vec_free (m->workers);
1621
1622   /* *INDENT-OFF* */
1623   pool_foreach (local, m->locals,
1624   ({
1625     vec_add1 (locals, local - m->locals);
1626     if (sm->num_workers > 1)
1627       {
1628         ip4_header_t ip;
1629         ip.src_address.as_u32 = local->addr.as_u32,
1630         bitmap = clib_bitmap_set (bitmap,
1631                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1632                                   1);
1633       }
1634   }));
1635   /* *INDENT-ON* */
1636
1637   ASSERT (vec_len (locals) > 1);
1638
1639   local = pool_elt_at_index (m->locals, locals[0]);
1640   local->prefix = local->probability;
1641   for (i = 1; i < vec_len (locals); i++)
1642     {
1643       local = pool_elt_at_index (m->locals, locals[i]);
1644       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1645       local->prefix = local->probability + prev_local->prefix;
1646     }
1647
1648   /* Assign workers */
1649   if (sm->num_workers > 1)
1650     {
1651       /* *INDENT-OFF* */
1652       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1653       /* *INDENT-ON* */
1654     }
1655
1656   return 0;
1657 }
1658
1659 int
1660 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1661                   u8 twice_nat)
1662 {
1663   snat_address_t *a = 0;
1664   snat_session_t *ses;
1665   u32 *ses_to_be_removed = 0, *ses_index;
1666   snat_main_per_thread_data_t *tsm;
1667   snat_static_mapping_t *m;
1668   snat_interface_t *interface;
1669   int i;
1670   snat_address_t *addresses =
1671     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1672
1673   /* Find SNAT address */
1674   for (i = 0; i < vec_len (addresses); i++)
1675     {
1676       if (addresses[i].addr.as_u32 == addr.as_u32)
1677         {
1678           a = addresses + i;
1679           break;
1680         }
1681     }
1682   if (!a)
1683     return VNET_API_ERROR_NO_SUCH_ENTRY;
1684
1685   if (delete_sm)
1686     {
1687       ip4_address_t pool_addr = { 0 };
1688       /* *INDENT-OFF* */
1689       pool_foreach (m, sm->static_mappings,
1690       ({
1691           if (m->external_addr.as_u32 == addr.as_u32)
1692             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1693                                             m->local_port, m->external_port,
1694                                             m->vrf_id,
1695                                             is_addr_only_static_mapping(m), ~0,
1696                                             m->proto, 0 /* is_add */,
1697                                             m->twice_nat,
1698                                             is_out2in_only_static_mapping(m),
1699                                             m->tag,
1700                                             is_identity_static_mapping(m),
1701                                             pool_addr, 0);
1702       }));
1703       /* *INDENT-ON* */
1704     }
1705   else
1706     {
1707       /* Check if address is used in some static mapping */
1708       if (is_snat_address_used_in_static_mapping (sm, addr))
1709         {
1710           nat_elog_notice ("address used in static mapping");
1711           return VNET_API_ERROR_UNSPECIFIED;
1712         }
1713     }
1714
1715   if (a->fib_index != ~0)
1716     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1717
1718   /* Delete sessions using address */
1719   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1720     {
1721       /* *INDENT-OFF* */
1722       vec_foreach (tsm, sm->per_thread_data)
1723         {
1724           pool_foreach (ses, tsm->sessions, ({
1725             if (ses->out2in.addr.as_u32 == addr.as_u32)
1726               {
1727                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1728                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1729               }
1730           }));
1731
1732           if (sm->endpoint_dependent){
1733               vec_foreach (ses_index, ses_to_be_removed)
1734                 {
1735                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1736                   nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1737                 }
1738           }else{
1739               vec_foreach (ses_index, ses_to_be_removed)
1740                 {
1741                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1742                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1743                 }
1744           }
1745
1746           vec_free (ses_to_be_removed);
1747         }
1748       /* *INDENT-ON* */
1749     }
1750
1751 #define _(N, i, n, s) \
1752   vec_free (a->busy_##n##_ports_per_thread);
1753   foreach_nat_protocol
1754 #undef _
1755     if (twice_nat)
1756     {
1757       vec_del1 (sm->twice_nat_addresses, i);
1758       return 0;
1759     }
1760   else
1761     vec_del1 (sm->addresses, i);
1762
1763   /* Delete external address from FIB */
1764   /* *INDENT-OFF* */
1765   pool_foreach (interface, sm->interfaces,
1766   ({
1767     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1768       continue;
1769
1770     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1771     break;
1772   }));
1773   pool_foreach (interface, sm->output_feature_interfaces,
1774   ({
1775     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1776       continue;
1777
1778     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1779     break;
1780   }));
1781   /* *INDENT-ON* */
1782
1783   return 0;
1784 }
1785
1786 static void
1787 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1788 {
1789 #define _(x)                                                                  \
1790   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1791                                 sw_if_index);                                 \
1792   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1793   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1794                                 sw_if_index);                                 \
1795   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1796   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1797                                 sw_if_index);                                 \
1798   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1799   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1800                                 sw_if_index);                                 \
1801   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1802   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1803                                 sw_if_index);                                 \
1804   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1805   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1806                                 sw_if_index);                                 \
1807   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1808   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1809                                 sw_if_index);                                 \
1810   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1811   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1812                                 sw_if_index);                                 \
1813   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1814   foreach_nat_counter;
1815 #undef _
1816   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1817   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1818 }
1819
1820 void
1821 expire_per_vrf_sessions (u32 fib_index)
1822 {
1823   per_vrf_sessions_t *per_vrf_sessions;
1824   snat_main_per_thread_data_t *tsm;
1825   snat_main_t *sm = &snat_main;
1826
1827   /* *INDENT-OFF* */
1828   vec_foreach (tsm, sm->per_thread_data)
1829     {
1830       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1831         {
1832           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1833               (per_vrf_sessions->tx_fib_index == fib_index))
1834             {
1835               per_vrf_sessions->expired = 1;
1836             }
1837         }
1838     }
1839   /* *INDENT-ON* */
1840 }
1841
1842 void
1843 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1844 {
1845   snat_main_t *sm = &snat_main;
1846   nat_fib_t *fib;
1847
1848   // we don't care if it is outside/inside fib
1849   // we just care about their ref_count
1850   // if it reaches 0 sessions should expire
1851   // because the fib isn't valid for NAT anymore
1852
1853   vec_foreach (fib, sm->fibs)
1854   {
1855     if (fib->fib_index == fib_index)
1856       {
1857         if (is_del)
1858           {
1859             fib->ref_count--;
1860             if (!fib->ref_count)
1861               {
1862                 vec_del1 (sm->fibs, fib - sm->fibs);
1863                 expire_per_vrf_sessions (fib_index);
1864               }
1865             return;
1866           }
1867         else
1868           fib->ref_count++;
1869       }
1870   }
1871   if (!is_del)
1872     {
1873       vec_add2 (sm->fibs, fib, 1);
1874       fib->ref_count = 1;
1875       fib->fib_index = fib_index;
1876     }
1877 }
1878
1879 int
1880 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1881 {
1882   snat_main_t *sm = &snat_main;
1883   snat_interface_t *i;
1884   const char *feature_name, *del_feature_name;
1885   snat_address_t *ap;
1886   snat_static_mapping_t *m;
1887   nat_outside_fib_t *outside_fib;
1888   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1889                                                        sw_if_index);
1890
1891   if (sm->out2in_dpo && !is_inside)
1892     return VNET_API_ERROR_UNSUPPORTED;
1893
1894   /* *INDENT-OFF* */
1895   pool_foreach (i, sm->output_feature_interfaces,
1896   ({
1897     if (i->sw_if_index == sw_if_index)
1898       return VNET_API_ERROR_VALUE_EXIST;
1899   }));
1900   /* *INDENT-ON* */
1901
1902   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1903     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1904   else
1905     {
1906       if (sm->num_workers > 1)
1907         feature_name =
1908           is_inside ? "nat44-in2out-worker-handoff" :
1909           "nat44-out2in-worker-handoff";
1910       else if (sm->endpoint_dependent)
1911         {
1912           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1913         }
1914       else
1915         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1916     }
1917
1918   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
1919     sm->fq_in2out_index =
1920       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1921
1922   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1923     sm->fq_out2in_index =
1924       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1925
1926   if (sm->endpoint_dependent)
1927     update_per_vrf_sessions_vec (fib_index, is_del);
1928
1929   if (!is_inside)
1930     {
1931       /* *INDENT-OFF* */
1932       vec_foreach (outside_fib, sm->outside_fibs)
1933         {
1934           if (outside_fib->fib_index == fib_index)
1935             {
1936               if (is_del)
1937                 {
1938                   outside_fib->refcount--;
1939                   if (!outside_fib->refcount)
1940                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1941                 }
1942               else
1943                 outside_fib->refcount++;
1944               goto feature_set;
1945             }
1946         }
1947       /* *INDENT-ON* */
1948       if (!is_del)
1949         {
1950           vec_add2 (sm->outside_fibs, outside_fib, 1);
1951           outside_fib->refcount = 1;
1952           outside_fib->fib_index = fib_index;
1953         }
1954     }
1955
1956 feature_set:
1957   /* *INDENT-OFF* */
1958   pool_foreach (i, sm->interfaces,
1959   ({
1960     if (i->sw_if_index == sw_if_index)
1961       {
1962         if (is_del)
1963           {
1964             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1965               {
1966                 if (is_inside)
1967                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1968                 else
1969                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1970
1971                 if (sm->num_workers > 1)
1972                   {
1973                     del_feature_name = "nat44-handoff-classify";
1974                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1975                                                  "nat44-out2in-worker-handoff";
1976                   }
1977                 else if (sm->endpoint_dependent)
1978                   {
1979                     del_feature_name = "nat44-ed-classify";
1980                     feature_name = !is_inside ?  "nat-pre-in2out" :
1981                                                  "nat-pre-out2in";
1982                   }
1983                 else
1984                   {
1985                     del_feature_name = "nat44-classify";
1986                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1987                   }
1988
1989                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1990                 if (rv)
1991                   return rv;
1992                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1993                                              sw_if_index, 0, 0, 0);
1994                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1995                                              sw_if_index, 1, 0, 0);
1996                 if (!is_inside)
1997                   {
1998                     if (sm->endpoint_dependent)
1999                       vnet_feature_enable_disable ("ip4-local",
2000                                                    "nat44-ed-hairpinning",
2001                                                    sw_if_index, 1, 0, 0);
2002                     else
2003                       vnet_feature_enable_disable ("ip4-local",
2004                                                    "nat44-hairpinning",
2005                                                    sw_if_index, 1, 0, 0);
2006                   }
2007               }
2008             else
2009               {
2010                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2011                 if (rv)
2012                   return rv;
2013                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2014                                              sw_if_index, 0, 0, 0);
2015                 pool_put (sm->interfaces, i);
2016                 if (is_inside)
2017                   {
2018                     if (sm->endpoint_dependent)
2019                       vnet_feature_enable_disable ("ip4-local",
2020                                                    "nat44-ed-hairpinning",
2021                                                    sw_if_index, 0, 0, 0);
2022                     else
2023                       vnet_feature_enable_disable ("ip4-local",
2024                                                    "nat44-hairpinning",
2025                                                    sw_if_index, 0, 0, 0);
2026                   }
2027               }
2028           }
2029         else
2030           {
2031             if ((nat_interface_is_inside(i) && is_inside) ||
2032                 (nat_interface_is_outside(i) && !is_inside))
2033               return 0;
2034
2035             if (sm->num_workers > 1)
2036               {
2037                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2038                                                  "nat44-out2in-worker-handoff";
2039                 feature_name = "nat44-handoff-classify";
2040               }
2041             else if (sm->endpoint_dependent)
2042               {
2043                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2044                                                  "nat-pre-out2in";
2045
2046                 feature_name = "nat44-ed-classify";
2047               }
2048             else
2049               {
2050                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2051                 feature_name = "nat44-classify";
2052               }
2053
2054             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2055             if (rv)
2056               return rv;
2057             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2058                                          sw_if_index, 0, 0, 0);
2059             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2060                                          sw_if_index, 1, 0, 0);
2061             if (!is_inside)
2062               {
2063                 if (sm->endpoint_dependent)
2064                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2065                                                sw_if_index, 0, 0, 0);
2066                 else
2067                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2068                                                sw_if_index, 0, 0, 0);
2069               }
2070             goto set_flags;
2071           }
2072
2073         goto fib;
2074       }
2075   }));
2076   /* *INDENT-ON* */
2077
2078   if (is_del)
2079     return VNET_API_ERROR_NO_SUCH_ENTRY;
2080
2081   pool_get (sm->interfaces, i);
2082   i->sw_if_index = sw_if_index;
2083   i->flags = 0;
2084   nat_validate_counters (sm, sw_if_index);
2085
2086   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2087                                0);
2088
2089   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2090   if (rv)
2091     return rv;
2092
2093   if (is_inside && !sm->out2in_dpo)
2094     {
2095       if (sm->endpoint_dependent)
2096         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2097                                      sw_if_index, 1, 0, 0);
2098       else
2099         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2100                                      sw_if_index, 1, 0, 0);
2101     }
2102
2103 set_flags:
2104   if (is_inside)
2105     {
2106       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2107       return 0;
2108     }
2109   else
2110     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2111
2112   /* Add/delete external addresses to FIB */
2113 fib:
2114   /* *INDENT-OFF* */
2115   vec_foreach (ap, sm->addresses)
2116     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2117
2118   pool_foreach (m, sm->static_mappings,
2119   ({
2120     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2121       continue;
2122
2123     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2124   }));
2125   /* *INDENT-ON* */
2126
2127   return 0;
2128 }
2129
2130 int
2131 snat_interface_add_del_output_feature (u32 sw_if_index,
2132                                        u8 is_inside, int is_del)
2133 {
2134   snat_main_t *sm = &snat_main;
2135   snat_interface_t *i;
2136   snat_address_t *ap;
2137   snat_static_mapping_t *m;
2138   nat_outside_fib_t *outside_fib;
2139   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2140                                                        sw_if_index);
2141
2142   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2143     return VNET_API_ERROR_UNSUPPORTED;
2144
2145   /* *INDENT-OFF* */
2146   pool_foreach (i, sm->interfaces,
2147   ({
2148     if (i->sw_if_index == sw_if_index)
2149       return VNET_API_ERROR_VALUE_EXIST;
2150   }));
2151   /* *INDENT-ON* */
2152
2153   if (sm->endpoint_dependent)
2154     update_per_vrf_sessions_vec (fib_index, is_del);
2155
2156   if (!is_inside)
2157     {
2158       /* *INDENT-OFF* */
2159       vec_foreach (outside_fib, sm->outside_fibs)
2160         {
2161           if (outside_fib->fib_index == fib_index)
2162             {
2163               if (is_del)
2164                 {
2165                   outside_fib->refcount--;
2166                   if (!outside_fib->refcount)
2167                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2168                 }
2169               else
2170                 outside_fib->refcount++;
2171               goto feature_set;
2172             }
2173         }
2174       /* *INDENT-ON* */
2175       if (!is_del)
2176         {
2177           vec_add2 (sm->outside_fibs, outside_fib, 1);
2178           outside_fib->refcount = 1;
2179           outside_fib->fib_index = fib_index;
2180         }
2181     }
2182
2183 feature_set:
2184   if (is_inside)
2185     {
2186       if (sm->endpoint_dependent)
2187         {
2188           int rv =
2189             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2190           if (rv)
2191             return rv;
2192           rv =
2193             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2194                                                             !is_del);
2195           if (rv)
2196             return rv;
2197           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2198                                        sw_if_index, !is_del, 0, 0);
2199           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2200                                        sw_if_index, !is_del, 0, 0);
2201         }
2202       else
2203         {
2204           int rv =
2205             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2206           if (rv)
2207             return rv;
2208           rv =
2209             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2210                                                             !is_del);
2211           if (rv)
2212             return rv;
2213           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2214                                        sw_if_index, !is_del, 0, 0);
2215           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2216                                        sw_if_index, !is_del, 0, 0);
2217         }
2218       goto fq;
2219     }
2220
2221   if (sm->num_workers > 1)
2222     {
2223       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2224       if (rv)
2225         return rv;
2226       rv =
2227         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2228       if (rv)
2229         return rv;
2230       vnet_feature_enable_disable ("ip4-unicast",
2231                                    "nat44-out2in-worker-handoff",
2232                                    sw_if_index, !is_del, 0, 0);
2233       vnet_feature_enable_disable ("ip4-output",
2234                                    "nat44-in2out-output-worker-handoff",
2235                                    sw_if_index, !is_del, 0, 0);
2236     }
2237   else
2238     {
2239       if (sm->endpoint_dependent)
2240         {
2241           int rv =
2242             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2243           if (rv)
2244             return rv;
2245           rv =
2246             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2247                                                             !is_del);
2248           if (rv)
2249             return rv;
2250           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2251                                        sw_if_index, !is_del, 0, 0);
2252           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2253                                        sw_if_index, !is_del, 0, 0);
2254         }
2255       else
2256         {
2257           int rv =
2258             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2259           if (rv)
2260             return rv;
2261           rv =
2262             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2263                                                             !is_del);
2264           if (rv)
2265             return rv;
2266           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2267                                        sw_if_index, !is_del, 0, 0);
2268           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2269                                        sw_if_index, !is_del, 0, 0);
2270         }
2271     }
2272
2273 fq:
2274   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2275     sm->fq_in2out_output_index =
2276       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2277
2278   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2279     sm->fq_out2in_index =
2280       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2281
2282   /* *INDENT-OFF* */
2283   pool_foreach (i, sm->output_feature_interfaces,
2284   ({
2285     if (i->sw_if_index == sw_if_index)
2286       {
2287         if (is_del)
2288           pool_put (sm->output_feature_interfaces, i);
2289         else
2290           return VNET_API_ERROR_VALUE_EXIST;
2291
2292         goto fib;
2293       }
2294   }));
2295   /* *INDENT-ON* */
2296
2297   if (is_del)
2298     return VNET_API_ERROR_NO_SUCH_ENTRY;
2299
2300   pool_get (sm->output_feature_interfaces, i);
2301   i->sw_if_index = sw_if_index;
2302   i->flags = 0;
2303   nat_validate_counters (sm, sw_if_index);
2304   if (is_inside)
2305     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2306   else
2307     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2308
2309   /* Add/delete external addresses to FIB */
2310 fib:
2311   if (is_inside)
2312     return 0;
2313
2314   /* *INDENT-OFF* */
2315   vec_foreach (ap, sm->addresses)
2316     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2317
2318   pool_foreach (m, sm->static_mappings,
2319   ({
2320     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2321       continue;
2322
2323     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2324   }));
2325   /* *INDENT-ON* */
2326
2327   return 0;
2328 }
2329
2330 int
2331 snat_set_workers (uword * bitmap)
2332 {
2333   snat_main_t *sm = &snat_main;
2334   int i, j = 0;
2335
2336   if (sm->num_workers < 2)
2337     return VNET_API_ERROR_FEATURE_DISABLED;
2338
2339   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2340     return VNET_API_ERROR_INVALID_WORKER;
2341
2342   vec_free (sm->workers);
2343   /* *INDENT-OFF* */
2344   clib_bitmap_foreach (i, bitmap,
2345     ({
2346       vec_add1(sm->workers, i);
2347       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2348       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2349       j++;
2350     }));
2351   /* *INDENT-ON* */
2352
2353   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2354
2355   return 0;
2356 }
2357
2358 static void
2359 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2360                          u32 old_fib_index)
2361 {
2362   snat_main_t *sm = &snat_main;
2363   nat_outside_fib_t *outside_fib;
2364   snat_interface_t *i;
2365   u8 is_add = 1;
2366   u8 match = 0;
2367
2368   if (new_fib_index == old_fib_index)
2369     return;
2370
2371   if (!vec_len (sm->outside_fibs))
2372     return;
2373
2374   /* *INDENT-OFF* */
2375   pool_foreach (i, sm->interfaces,
2376     ({
2377       if (i->sw_if_index == sw_if_index)
2378         {
2379           if (!(nat_interface_is_outside (i)))
2380             return;
2381           match = 1;
2382         }
2383     }));
2384
2385   pool_foreach (i, sm->output_feature_interfaces,
2386     ({
2387       if (i->sw_if_index == sw_if_index)
2388         {
2389           if (!(nat_interface_is_outside (i)))
2390             return;
2391           match = 1;
2392         }
2393     }));
2394   /* *INDENT-ON* */
2395
2396   if (!match)
2397     return;
2398
2399   vec_foreach (outside_fib, sm->outside_fibs)
2400   {
2401     if (outside_fib->fib_index == old_fib_index)
2402       {
2403         outside_fib->refcount--;
2404         if (!outside_fib->refcount)
2405           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2406         break;
2407       }
2408   }
2409
2410   vec_foreach (outside_fib, sm->outside_fibs)
2411   {
2412     if (outside_fib->fib_index == new_fib_index)
2413       {
2414         outside_fib->refcount++;
2415         is_add = 0;
2416         break;
2417       }
2418   }
2419
2420   if (is_add)
2421     {
2422       vec_add2 (sm->outside_fibs, outside_fib, 1);
2423       outside_fib->refcount = 1;
2424       outside_fib->fib_index = new_fib_index;
2425     }
2426 }
2427
2428 static void
2429 snat_ip4_table_bind (ip4_main_t * im,
2430                      uword opaque,
2431                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2432 {
2433   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2434 }
2435
2436 static void
2437 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2438                                        uword opaque,
2439                                        u32 sw_if_index,
2440                                        ip4_address_t * address,
2441                                        u32 address_length,
2442                                        u32 if_address_index, u32 is_delete);
2443
2444 static void
2445 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2446                                  uword opaque,
2447                                  u32 sw_if_index,
2448                                  ip4_address_t * address,
2449                                  u32 address_length,
2450                                  u32 if_address_index, u32 is_delete);
2451
2452 static int
2453 nat_alloc_addr_and_port_default (snat_address_t * addresses, u32 fib_index,
2454                                  u32 thread_index, nat_protocol_t proto,
2455                                  ip4_address_t * addr, u16 * port,
2456                                  u16 port_per_thread, u32 snat_thread_index);
2457
2458 void
2459 test_key_calc_split ()
2460 {
2461   ip4_address_t l_addr;
2462   l_addr.as_u8[0] = 1;
2463   l_addr.as_u8[1] = 1;
2464   l_addr.as_u8[2] = 1;
2465   l_addr.as_u8[3] = 1;
2466   ip4_address_t r_addr;
2467   r_addr.as_u8[0] = 2;
2468   r_addr.as_u8[1] = 2;
2469   r_addr.as_u8[2] = 2;
2470   r_addr.as_u8[3] = 2;
2471   u16 l_port = 40001;
2472   u16 r_port = 40301;
2473   u8 proto = 9;
2474   u32 fib_index = 9000001;
2475   u32 thread_index = 3000000001;
2476   u32 session_index = 3000000221;
2477   clib_bihash_kv_16_8_t kv;
2478   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2479               thread_index, session_index);
2480   ip4_address_t l_addr2;
2481   ip4_address_t r_addr2;
2482   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2483   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2484   u16 l_port2 = 0;
2485   u16 r_port2 = 0;
2486   u8 proto2 = 0;
2487   u32 fib_index2 = 0;
2488   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2489                &r_port2);
2490   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2491   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2492   ASSERT (l_port == l_port2);
2493   ASSERT (r_port == r_port2);
2494   ASSERT (proto == proto2);
2495   ASSERT (fib_index == fib_index2);
2496   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2497   ASSERT (session_index == ed_value_get_session_index (&kv));
2498
2499   fib_index = 7001;
2500   proto = 5;
2501   nat_protocol_t proto3 = ~0;
2502   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2503   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2504   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2505   ASSERT (l_port == l_port2);
2506   ASSERT (proto == proto3);
2507   ASSERT (fib_index == fib_index2);
2508 }
2509
2510 static clib_error_t *
2511 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2512 {
2513   snat_main_t *sm = &snat_main;
2514   u32 fib_index;
2515
2516   if (sm->endpoint_dependent)
2517     {
2518       // TODO: consider removing all NAT interfaces
2519
2520       if (!is_add)
2521         {
2522           fib_index = ip4_fib_index_from_table_id (table_id);
2523           if (fib_index != ~0)
2524             expire_per_vrf_sessions (fib_index);
2525         }
2526     }
2527   return 0;
2528 }
2529
2530 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2531
2532
2533 static clib_error_t *
2534 snat_init (vlib_main_t * vm)
2535 {
2536   snat_main_t *sm = &snat_main;
2537   clib_error_t *error = 0;
2538   ip4_main_t *im = &ip4_main;
2539   ip_lookup_main_t *lm = &im->lookup_main;
2540   uword *p;
2541   vlib_thread_registration_t *tr;
2542   vlib_thread_main_t *tm = vlib_get_thread_main ();
2543   uword *bitmap = 0;
2544   u32 i;
2545   ip4_add_del_interface_address_callback_t cb4;
2546   vlib_node_t *node;
2547
2548   sm->vnet_main = vnet_get_main ();
2549   sm->ip4_main = im;
2550   sm->ip4_lookup_main = lm;
2551   sm->api_main = vlibapi_get_main ();
2552   sm->first_worker_index = 0;
2553   sm->num_workers = 0;
2554   sm->workers = 0;
2555   sm->port_per_thread = 0xffff - 1024;
2556   sm->fq_in2out_index = ~0;
2557   sm->fq_in2out_output_index = ~0;
2558   sm->fq_out2in_index = ~0;
2559
2560   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2561   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2562   sm->forwarding_enabled = 0;
2563   sm->log_class = vlib_log_register_class ("nat", 0);
2564   sm->log_level = SNAT_LOG_ERROR;
2565   sm->mss_clamping = 0;
2566
2567   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2568   sm->error_node_index = node->index;
2569
2570   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2571   sm->pre_in2out_node_index = node->index;
2572   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2573   sm->pre_out2in_node_index = node->index;
2574
2575   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2576   sm->pre_in2out_node_index = node->index;
2577
2578   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2579   sm->pre_out2in_node_index = node->index;
2580
2581   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2582   sm->in2out_node_index = node->index;
2583   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2584   sm->in2out_output_node_index = node->index;
2585   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2586   sm->in2out_fast_node_index = node->index;
2587   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2588   sm->in2out_slowpath_node_index = node->index;
2589   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2590   sm->in2out_slowpath_output_node_index = node->index;
2591
2592   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2593   sm->ed_in2out_node_index = node->index;
2594   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2595   sm->ed_in2out_slowpath_node_index = node->index;
2596
2597   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2598   sm->out2in_node_index = node->index;
2599   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2600   sm->out2in_fast_node_index = node->index;
2601
2602   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2603   sm->ed_out2in_node_index = node->index;
2604   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2605   sm->ed_out2in_slowpath_node_index = node->index;
2606
2607   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2608   sm->hairpinning_node_index = node->index;
2609   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2610   sm->hairpin_dst_node_index = node->index;
2611   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2612   sm->hairpin_src_node_index = node->index;
2613   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2614   sm->ed_hairpinning_node_index = node->index;
2615   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2616   sm->ed_hairpin_dst_node_index = node->index;
2617   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2618   sm->ed_hairpin_src_node_index = node->index;
2619
2620   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2621   if (p)
2622     {
2623       tr = (vlib_thread_registration_t *) p[0];
2624       if (tr)
2625         {
2626           sm->num_workers = tr->count;
2627           sm->first_worker_index = tr->first_index;
2628         }
2629     }
2630
2631   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2632
2633   /* Use all available workers by default */
2634   if (sm->num_workers > 1)
2635     {
2636       for (i = 0; i < sm->num_workers; i++)
2637         bitmap = clib_bitmap_set (bitmap, i, 1);
2638       snat_set_workers (bitmap);
2639       clib_bitmap_free (bitmap);
2640     }
2641   else
2642     {
2643       sm->per_thread_data[0].snat_thread_index = 0;
2644     }
2645
2646   error = snat_api_init (vm, sm);
2647   if (error)
2648     return error;
2649
2650   /* Set up the interface address add/del callback */
2651   cb4.function = snat_ip4_add_del_interface_address_cb;
2652   cb4.function_opaque = 0;
2653
2654   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2655
2656   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2657   cb4.function_opaque = 0;
2658
2659   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2660
2661   nat_dpo_module_init ();
2662
2663   /* Init counters */
2664   sm->total_users.name = "total-users";
2665   sm->total_users.stat_segment_name = "/nat44/total-users";
2666   vlib_validate_simple_counter (&sm->total_users, 0);
2667   vlib_zero_simple_counter (&sm->total_users, 0);
2668   sm->total_sessions.name = "total-sessions";
2669   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2670   vlib_validate_simple_counter (&sm->total_sessions, 0);
2671   vlib_zero_simple_counter (&sm->total_sessions, 0);
2672   sm->user_limit_reached.name = "user-limit-reached";
2673   sm->user_limit_reached.stat_segment_name = "/nat44/user-limit-reached";
2674   vlib_validate_simple_counter (&sm->user_limit_reached, 0);
2675   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2676
2677 #define _(x)                                            \
2678   sm->counters.fastpath.in2out.x.name = #x;             \
2679   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2680       "/nat44/in2out/fastpath/" #x;                     \
2681   sm->counters.slowpath.in2out.x.name = #x;             \
2682   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2683       "/nat44/in2out/slowpath/" #x;                     \
2684   sm->counters.fastpath.out2in.x.name = #x;             \
2685   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2686       "/nat44/out2in/fastpath/" #x;                     \
2687   sm->counters.slowpath.out2in.x.name = #x;             \
2688   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2689       "/nat44/out2in/slowpath/" #x;                     \
2690   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2691   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2692       "/nat44/ed/in2out/fastpath/" #x;                  \
2693   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2694   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2695       "/nat44/ed/in2out/slowpath/" #x;                  \
2696   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2697   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2698       "/nat44/ed/out2in/fastpath/" #x;                  \
2699   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2700   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2701       "/nat44/ed/out2in/slowpath/" #x;
2702   foreach_nat_counter;
2703 #undef _
2704   sm->counters.hairpinning.name = "hairpinning";
2705   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2706
2707   /* Init IPFIX logging */
2708   snat_ipfix_logging_init (vm);
2709
2710   /* Init NAT64 */
2711   error = nat64_init (vm);
2712   if (error)
2713     return error;
2714
2715   ip4_table_bind_callback_t cbt4 = {
2716     .function = snat_ip4_table_bind,
2717   };
2718   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2719
2720   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2721                                         FIB_SOURCE_PRIORITY_HI,
2722                                         FIB_SOURCE_BH_SIMPLE);
2723   nat_fib_src_low = fib_source_allocate ("nat-low",
2724                                          FIB_SOURCE_PRIORITY_LOW,
2725                                          FIB_SOURCE_BH_SIMPLE);
2726
2727   test_key_calc_split ();
2728   return error;
2729 }
2730
2731 VLIB_INIT_FUNCTION (snat_init);
2732
2733 void
2734 snat_free_outside_address_and_port (snat_address_t * addresses,
2735                                     u32 thread_index,
2736                                     ip4_address_t * addr,
2737                                     u16 port, nat_protocol_t protocol)
2738 {
2739   snat_address_t *a;
2740   u32 address_index;
2741   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2742
2743   for (address_index = 0; address_index < vec_len (addresses);
2744        address_index++)
2745     {
2746       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2747         break;
2748     }
2749
2750   ASSERT (address_index < vec_len (addresses));
2751
2752   a = addresses + address_index;
2753
2754   switch (protocol)
2755     {
2756 #define _(N, i, n, s) \
2757     case NAT_PROTOCOL_##N: \
2758       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2759       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2760       a->busy_##n##_ports--; \
2761       a->busy_##n##_ports_per_thread[thread_index]--; \
2762       break;
2763       foreach_nat_protocol
2764 #undef _
2765     default:
2766       nat_elog_info ("unknown protocol");
2767       return;
2768     }
2769 }
2770
2771 static int
2772 nat_set_outside_address_and_port (snat_address_t * addresses,
2773                                   u32 thread_index, ip4_address_t addr,
2774                                   u16 port, nat_protocol_t protocol)
2775 {
2776   snat_address_t *a = 0;
2777   u32 address_index;
2778   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2779
2780   for (address_index = 0; address_index < vec_len (addresses);
2781        address_index++)
2782     {
2783       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2784         continue;
2785
2786       a = addresses + address_index;
2787       switch (protocol)
2788         {
2789 #define _(N, j, n, s) \
2790         case NAT_PROTOCOL_##N: \
2791           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2792             return VNET_API_ERROR_INSTANCE_IN_USE; \
2793           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2794           a->busy_##n##_ports_per_thread[thread_index]++; \
2795           a->busy_##n##_ports++; \
2796           return 0;
2797           foreach_nat_protocol
2798 #undef _
2799         default:
2800           nat_elog_info ("unknown protocol");
2801           return 1;
2802         }
2803     }
2804
2805   return VNET_API_ERROR_NO_SUCH_ENTRY;
2806 }
2807
2808 int
2809 snat_static_mapping_match (snat_main_t * sm,
2810                            ip4_address_t match_addr,
2811                            u16 match_port,
2812                            u32 match_fib_index,
2813                            nat_protocol_t match_protocol,
2814                            ip4_address_t * mapping_addr,
2815                            u16 * mapping_port,
2816                            u32 * mapping_fib_index,
2817                            u8 by_external,
2818                            u8 * is_addr_only,
2819                            twice_nat_type_t * twice_nat,
2820                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2821                            u8 * is_identity_nat, snat_static_mapping_t ** out)
2822 {
2823   clib_bihash_kv_8_8_t kv, value;
2824   clib_bihash_8_8_t *mapping_hash;
2825   snat_static_mapping_t *m;
2826   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2827   nat44_lb_addr_port_t *local;
2828   u8 backend_index;
2829
2830   if (!by_external)
2831     {
2832       mapping_hash = &sm->static_mapping_by_local;
2833       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2834                   match_protocol);
2835       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2836         {
2837           /* Try address only mapping */
2838           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2839           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2840             return 1;
2841         }
2842     }
2843   else
2844     {
2845       mapping_hash = &sm->static_mapping_by_external;
2846       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2847       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2848         {
2849           /* Try address only mapping */
2850           init_nat_k (&kv, match_addr, 0, 0, 0);
2851           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2852             return 1;
2853         }
2854     }
2855
2856   m = pool_elt_at_index (sm->static_mappings, value.value);
2857
2858   if (by_external)
2859     {
2860       if (is_lb_static_mapping (m))
2861         {
2862           if (PREDICT_FALSE (lb != 0))
2863             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2864           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2865                                                           match_addr,
2866                                                           match_protocol,
2867                                                           match_port,
2868                                                           &backend_index))
2869             {
2870               local = pool_elt_at_index (m->locals, backend_index);
2871               *mapping_addr = local->addr;
2872               *mapping_port = local->port;
2873               *mapping_fib_index = local->fib_index;
2874               goto end;
2875             }
2876           // pick locals matching this worker
2877           if (PREDICT_FALSE (sm->num_workers > 1))
2878             {
2879               u32 thread_index = vlib_get_thread_index ();
2880               /* *INDENT-OFF* */
2881               pool_foreach_index (i, m->locals,
2882               ({
2883                 local = pool_elt_at_index (m->locals, i);
2884
2885                 ip4_header_t ip = {
2886                   .src_address = local->addr,
2887                 };
2888
2889                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2890                     thread_index)
2891                   {
2892                     vec_add1 (tmp, i);
2893                   }
2894               }));
2895               /* *INDENT-ON* */
2896               ASSERT (vec_len (tmp) != 0);
2897             }
2898           else
2899             {
2900               /* *INDENT-OFF* */
2901               pool_foreach_index (i, m->locals,
2902               ({
2903                 vec_add1 (tmp, i);
2904               }));
2905               /* *INDENT-ON* */
2906             }
2907           hi = vec_len (tmp) - 1;
2908           local = pool_elt_at_index (m->locals, tmp[hi]);
2909           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2910           while (lo < hi)
2911             {
2912               mid = ((hi - lo) >> 1) + lo;
2913               local = pool_elt_at_index (m->locals, tmp[mid]);
2914               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2915             }
2916           local = pool_elt_at_index (m->locals, tmp[lo]);
2917           if (!(local->prefix >= rand))
2918             return 1;
2919           *mapping_addr = local->addr;
2920           *mapping_port = local->port;
2921           *mapping_fib_index = local->fib_index;
2922           if (m->affinity)
2923             {
2924               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2925                                                 match_protocol, match_port,
2926                                                 tmp[lo], m->affinity,
2927                                                 m->affinity_per_service_list_head_index))
2928                 nat_elog_info ("create affinity record failed");
2929             }
2930           vec_free (tmp);
2931         }
2932       else
2933         {
2934           if (PREDICT_FALSE (lb != 0))
2935             *lb = NO_LB_NAT;
2936           *mapping_fib_index = m->fib_index;
2937           *mapping_addr = m->local_addr;
2938           /* Address only mapping doesn't change port */
2939           *mapping_port = is_addr_only_static_mapping (m) ? match_port
2940             : m->local_port;
2941         }
2942     }
2943   else
2944     {
2945       *mapping_addr = m->external_addr;
2946       /* Address only mapping doesn't change port */
2947       *mapping_port = is_addr_only_static_mapping (m) ? match_port
2948         : m->external_port;
2949       *mapping_fib_index = sm->outside_fib_index;
2950     }
2951
2952 end:
2953   if (PREDICT_FALSE (is_addr_only != 0))
2954     *is_addr_only = is_addr_only_static_mapping (m);
2955
2956   if (PREDICT_FALSE (twice_nat != 0))
2957     *twice_nat = m->twice_nat;
2958
2959   if (PREDICT_FALSE (is_identity_nat != 0))
2960     *is_identity_nat = is_identity_static_mapping (m);
2961
2962   if (out != 0)
2963     *out = m;
2964
2965   return 0;
2966 }
2967
2968 int
2969 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2970                                      u32 fib_index,
2971                                      u32 thread_index,
2972                                      nat_protocol_t proto,
2973                                      ip4_address_t * addr,
2974                                      u16 * port,
2975                                      u16 port_per_thread,
2976                                      u32 snat_thread_index)
2977 {
2978   snat_main_t *sm = &snat_main;
2979
2980   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, proto,
2981                                   addr, port, port_per_thread,
2982                                   snat_thread_index);
2983 }
2984
2985 static int
2986 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2987                                  u32 fib_index,
2988                                  u32 thread_index,
2989                                  nat_protocol_t proto,
2990                                  ip4_address_t * addr,
2991                                  u16 * port,
2992                                  u16 port_per_thread, u32 snat_thread_index)
2993 {
2994   int i;
2995   snat_address_t *a, *ga = 0;
2996   u32 portnum;
2997
2998   for (i = 0; i < vec_len (addresses); i++)
2999     {
3000       a = addresses + i;
3001       switch (proto)
3002         {
3003 #define _(N, j, n, s) \
3004         case NAT_PROTOCOL_##N: \
3005           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
3006             { \
3007               if (a->fib_index == fib_index) \
3008                 { \
3009                   while (1) \
3010                     { \
3011                       portnum = (port_per_thread * \
3012                         snat_thread_index) + \
3013                         snat_random_port(0, port_per_thread - 1) + 1024; \
3014                       if (a->busy_##n##_port_refcounts[portnum]) \
3015                         continue; \
3016                       --a->busy_##n##_port_refcounts[portnum]; \
3017                       a->busy_##n##_ports_per_thread[thread_index]++; \
3018                       a->busy_##n##_ports++; \
3019                       *addr = a->addr; \
3020                       *port = clib_host_to_net_u16(portnum); \
3021                       return 0; \
3022                     } \
3023                 } \
3024               else if (a->fib_index == ~0) \
3025                 { \
3026                   ga = a; \
3027                 } \
3028             } \
3029           break;
3030           foreach_nat_protocol
3031 #undef _
3032         default:
3033           nat_elog_info ("unknown protocol");
3034           return 1;
3035         }
3036
3037     }
3038
3039   if (ga)
3040     {
3041       a = ga;
3042       switch (proto)
3043         {
3044 #define _(N, j, n, s) \
3045         case NAT_PROTOCOL_##N: \
3046           while (1) \
3047             { \
3048               portnum = (port_per_thread * \
3049                 snat_thread_index) + \
3050                 snat_random_port(0, port_per_thread - 1) + 1024; \
3051               if (a->busy_##n##_port_refcounts[portnum]) \
3052                 continue; \
3053               ++a->busy_##n##_port_refcounts[portnum]; \
3054               a->busy_##n##_ports_per_thread[thread_index]++; \
3055               a->busy_##n##_ports++; \
3056               *addr = a->addr; \
3057               *port = clib_host_to_net_u16(portnum); \
3058               return 0; \
3059             }
3060           break;
3061           foreach_nat_protocol
3062 #undef _
3063         default:
3064           nat_elog_info ("unknown protocol");
3065           return 1;
3066         }
3067     }
3068
3069   /* Totally out of translations to use... */
3070   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3071   return 1;
3072 }
3073
3074 static int
3075 nat_alloc_addr_and_port_mape (snat_address_t * addresses, u32 fib_index,
3076                               u32 thread_index, nat_protocol_t proto,
3077                               ip4_address_t * addr, u16 * port,
3078                               u16 port_per_thread, u32 snat_thread_index)
3079 {
3080   snat_main_t *sm = &snat_main;
3081   snat_address_t *a = addresses;
3082   u16 m, ports, portnum, A, j;
3083   m = 16 - (sm->psid_offset + sm->psid_length);
3084   ports = (1 << (16 - sm->psid_length)) - (1 << m);
3085
3086   if (!vec_len (addresses))
3087     goto exhausted;
3088
3089   switch (proto)
3090     {
3091 #define _(N, i, n, s) \
3092     case NAT_PROTOCOL_##N: \
3093       if (a->busy_##n##_ports < ports) \
3094         { \
3095           while (1) \
3096             { \
3097               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3098               j = snat_random_port(0, pow2_mask(m)); \
3099               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3100               if (a->busy_##n##_port_refcounts[portnum]) \
3101                 continue; \
3102               ++a->busy_##n##_port_refcounts[portnum]; \
3103               a->busy_##n##_ports++; \
3104               *addr = a->addr; \
3105               *port = clib_host_to_net_u16 (portnum); \
3106               return 0; \
3107             } \
3108         } \
3109       break;
3110       foreach_nat_protocol
3111 #undef _
3112     default:
3113       nat_elog_info ("unknown protocol");
3114       return 1;
3115     }
3116
3117 exhausted:
3118   /* Totally out of translations to use... */
3119   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3120   return 1;
3121 }
3122
3123 static int
3124 nat_alloc_addr_and_port_range (snat_address_t * addresses, u32 fib_index,
3125                                u32 thread_index, nat_protocol_t proto,
3126                                ip4_address_t * addr, u16 * port,
3127                                u16 port_per_thread, u32 snat_thread_index)
3128 {
3129   snat_main_t *sm = &snat_main;
3130   snat_address_t *a = addresses;
3131   u16 portnum, ports;
3132
3133   ports = sm->end_port - sm->start_port + 1;
3134
3135   if (!vec_len (addresses))
3136     goto exhausted;
3137
3138   switch (proto)
3139     {
3140 #define _(N, i, n, s) \
3141     case NAT_PROTOCOL_##N: \
3142       if (a->busy_##n##_ports < ports) \
3143         { \
3144           while (1) \
3145             { \
3146               portnum = snat_random_port(sm->start_port, sm->end_port); \
3147               if (a->busy_##n##_port_refcounts[portnum]) \
3148                 continue; \
3149               ++a->busy_##n##_port_refcounts[portnum]; \
3150               a->busy_##n##_ports++; \
3151               *addr = a->addr; \
3152               *port = clib_host_to_net_u16 (portnum); \
3153               return 0; \
3154             } \
3155         } \
3156       break;
3157       foreach_nat_protocol
3158 #undef _
3159     default:
3160       nat_elog_info ("unknown protocol");
3161       return 1;
3162     }
3163
3164 exhausted:
3165   /* Totally out of translations to use... */
3166   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3167   return 1;
3168 }
3169
3170 void
3171 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3172 {
3173   dpo_id_t dpo_v4 = DPO_INVALID;
3174   fib_prefix_t pfx = {
3175     .fp_proto = FIB_PROTOCOL_IP4,
3176     .fp_len = 32,
3177     .fp_addr.ip4.as_u32 = addr.as_u32,
3178   };
3179
3180   if (is_add)
3181     {
3182       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3183       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3184                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3185       dpo_reset (&dpo_v4);
3186     }
3187   else
3188     {
3189       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3190     }
3191 }
3192
3193 u8 *
3194 format_session_kvp (u8 * s, va_list * args)
3195 {
3196   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3197
3198   s = format (s, "%U session-index %llu", format_snat_key, v->key, v->value);
3199
3200   return s;
3201 }
3202
3203 u8 *
3204 format_static_mapping_kvp (u8 * s, va_list * args)
3205 {
3206   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3207
3208   s = format (s, "%U static-mapping-index %llu",
3209               format_snat_key, v->key, v->value);
3210
3211   return s;
3212 }
3213
3214 u8 *
3215 format_user_kvp (u8 * s, va_list * args)
3216 {
3217   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3218   snat_user_key_t k;
3219
3220   k.as_u64 = v->key;
3221
3222   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3223               k.fib_index, v->value);
3224
3225   return s;
3226 }
3227
3228 u8 *
3229 format_ed_session_kvp (u8 * s, va_list * args)
3230 {
3231   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3232
3233   u8 proto;
3234   u16 r_port, l_port;
3235   ip4_address_t l_addr, r_addr;
3236   u32 fib_index;
3237
3238   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
3239   s =
3240     format (s,
3241             "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
3242             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
3243             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
3244             format_ip_protocol, proto, fib_index,
3245             ed_value_get_session_index (v), ed_value_get_thread_index (v));
3246
3247   return s;
3248 }
3249
3250 static u32
3251 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3252                            u8 is_output)
3253 {
3254   snat_main_t *sm = &snat_main;
3255   u32 next_worker_index = 0;
3256   u32 hash;
3257
3258   next_worker_index = sm->first_worker_index;
3259   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3260     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3261
3262   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3263     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3264   else
3265     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3266
3267   return next_worker_index;
3268 }
3269
3270 static u32
3271 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3272                            u32 rx_fib_index0, u8 is_output)
3273 {
3274   snat_main_t *sm = &snat_main;
3275   udp_header_t *udp;
3276   u16 port;
3277   clib_bihash_kv_8_8_t kv, value;
3278   snat_static_mapping_t *m;
3279   u32 proto;
3280   u32 next_worker_index = 0;
3281
3282   /* first try static mappings without port */
3283   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3284     {
3285       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
3286       if (!clib_bihash_search_8_8
3287           (&sm->static_mapping_by_external, &kv, &value))
3288         {
3289           m = pool_elt_at_index (sm->static_mappings, value.value);
3290           return m->workers[0];
3291         }
3292     }
3293
3294   proto = ip_proto_to_nat_proto (ip0->protocol);
3295   udp = ip4_next_header (ip0);
3296   port = udp->dst_port;
3297
3298   /* unknown protocol */
3299   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3300     {
3301       /* use current thread */
3302       return vlib_get_thread_index ();
3303     }
3304
3305   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3306     {
3307       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3308       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3309       if (!icmp_type_is_error_message
3310           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3311         port = vnet_buffer (b)->ip.reass.l4_src_port;
3312       else
3313         {
3314           /* if error message, then it's not fragmented and we can access it */
3315           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3316           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3317           void *l4_header = ip4_next_header (inner_ip);
3318           switch (proto)
3319             {
3320             case NAT_PROTOCOL_ICMP:
3321               icmp = (icmp46_header_t *) l4_header;
3322               echo = (icmp_echo_header_t *) (icmp + 1);
3323               port = echo->identifier;
3324               break;
3325             case NAT_PROTOCOL_UDP:
3326             case NAT_PROTOCOL_TCP:
3327               port = ((tcp_udp_header_t *) l4_header)->src_port;
3328               break;
3329             default:
3330               return vlib_get_thread_index ();
3331             }
3332         }
3333     }
3334
3335   /* try static mappings with port */
3336   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3337     {
3338       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
3339       if (!clib_bihash_search_8_8
3340           (&sm->static_mapping_by_external, &kv, &value))
3341         {
3342           m = pool_elt_at_index (sm->static_mappings, value.value);
3343           return m->workers[0];
3344         }
3345     }
3346
3347   /* worker by outside port */
3348   next_worker_index = sm->first_worker_index;
3349   next_worker_index +=
3350     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3351   return next_worker_index;
3352 }
3353
3354 static u32
3355 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3356                                u8 is_output)
3357 {
3358   snat_main_t *sm = &snat_main;
3359   u32 next_worker_index = sm->first_worker_index;
3360   u32 hash;
3361
3362   clib_bihash_kv_16_8_t kv16, value16;
3363   snat_main_per_thread_data_t *tsm;
3364   udp_header_t *udp;
3365
3366   if (PREDICT_FALSE (is_output))
3367     {
3368       u32 fib_index = sm->outside_fib_index;
3369       nat_outside_fib_t *outside_fib;
3370       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3371       fib_prefix_t pfx = {
3372         .fp_proto = FIB_PROTOCOL_IP4,
3373         .fp_len = 32,
3374         .fp_addr = {
3375                     .ip4.as_u32 = ip->dst_address.as_u32,
3376                     }
3377         ,
3378       };
3379
3380       udp = ip4_next_header (ip);
3381
3382       switch (vec_len (sm->outside_fibs))
3383         {
3384         case 0:
3385           fib_index = sm->outside_fib_index;
3386           break;
3387         case 1:
3388           fib_index = sm->outside_fibs[0].fib_index;
3389           break;
3390         default:
3391             /* *INDENT-OFF* */
3392             vec_foreach (outside_fib, sm->outside_fibs)
3393               {
3394                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3395                 if (FIB_NODE_INDEX_INVALID != fei)
3396                   {
3397                     if (fib_entry_get_resolving_interface (fei) != ~0)
3398                       {
3399                         fib_index = outside_fib->fib_index;
3400                         break;
3401                       }
3402                   }
3403               }
3404             /* *INDENT-ON* */
3405           break;
3406         }
3407
3408       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3409                  udp->dst_port, fib_index, ip->protocol);
3410
3411       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3412                                                   &kv16, &value16)))
3413         {
3414           tsm =
3415             vec_elt_at_index (sm->per_thread_data,
3416                               ed_value_get_thread_index (&value16));
3417           next_worker_index += tsm->thread_index;
3418
3419           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3420                                   next_worker_index, fib_index,
3421                                   clib_net_to_host_u32 (ip->
3422                                                         src_address.as_u32),
3423                                   clib_net_to_host_u32 (ip->
3424                                                         dst_address.as_u32));
3425
3426           return next_worker_index;
3427         }
3428     }
3429
3430   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3431     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3432
3433   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3434     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3435   else
3436     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3437
3438   if (PREDICT_TRUE (!is_output))
3439     {
3440       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3441                               next_worker_index, rx_fib_index,
3442                               clib_net_to_host_u32 (ip->src_address.as_u32),
3443                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3444     }
3445   else
3446     {
3447       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3448                               next_worker_index, rx_fib_index,
3449                               clib_net_to_host_u32 (ip->src_address.as_u32),
3450                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3451     }
3452
3453   return next_worker_index;
3454 }
3455
3456 static u32
3457 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3458                                u32 rx_fib_index, u8 is_output)
3459 {
3460   snat_main_t *sm = &snat_main;
3461   clib_bihash_kv_8_8_t kv, value;
3462   clib_bihash_kv_16_8_t kv16, value16;
3463   snat_main_per_thread_data_t *tsm;
3464
3465   u32 proto, next_worker_index = 0;
3466   udp_header_t *udp;
3467   u16 port;
3468   snat_static_mapping_t *m;
3469   u32 hash;
3470
3471   proto = ip_proto_to_nat_proto (ip->protocol);
3472
3473   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3474     {
3475       udp = ip4_next_header (ip);
3476
3477       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3478                  udp->src_port, rx_fib_index, ip->protocol);
3479
3480       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3481                                                   &kv16, &value16)))
3482         {
3483           tsm =
3484             vec_elt_at_index (sm->per_thread_data,
3485                               ed_value_get_thread_index (&value16));
3486           vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
3487             ed_value_get_session_index (&value16);
3488           next_worker_index = sm->first_worker_index + tsm->thread_index;
3489           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3490                                   next_worker_index, rx_fib_index,
3491                                   clib_net_to_host_u32 (ip->
3492                                                         src_address.as_u32),
3493                                   clib_net_to_host_u32 (ip->
3494                                                         dst_address.as_u32));
3495           return next_worker_index;
3496         }
3497     }
3498   else if (proto == NAT_PROTOCOL_ICMP)
3499     {
3500       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
3501         {
3502           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3503                                                       &kv16, &value16)))
3504             {
3505               tsm =
3506                 vec_elt_at_index (sm->per_thread_data,
3507                                   ed_value_get_thread_index (&value16));
3508               next_worker_index = sm->first_worker_index + tsm->thread_index;
3509               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3510                                       next_worker_index, rx_fib_index,
3511                                       clib_net_to_host_u32 (ip->
3512                                                             src_address.as_u32),
3513                                       clib_net_to_host_u32 (ip->
3514                                                             dst_address.as_u32));
3515               return next_worker_index;
3516             }
3517         }
3518     }
3519
3520   /* first try static mappings without port */
3521   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3522     {
3523       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3524       if (!clib_bihash_search_8_8
3525           (&sm->static_mapping_by_external, &kv, &value))
3526         {
3527           m = pool_elt_at_index (sm->static_mappings, value.value);
3528           next_worker_index = m->workers[0];
3529           goto done;
3530         }
3531     }
3532
3533   /* unknown protocol */
3534   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3535     {
3536       /* use current thread */
3537       next_worker_index = vlib_get_thread_index ();
3538       goto done;
3539     }
3540
3541   udp = ip4_next_header (ip);
3542   port = udp->dst_port;
3543
3544   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3545     {
3546       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3547       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3548       if (!icmp_type_is_error_message
3549           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3550         port = vnet_buffer (b)->ip.reass.l4_src_port;
3551       else
3552         {
3553           /* if error message, then it's not fragmented and we can access it */
3554           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3555           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3556           void *l4_header = ip4_next_header (inner_ip);
3557           switch (proto)
3558             {
3559             case NAT_PROTOCOL_ICMP:
3560               icmp = (icmp46_header_t *) l4_header;
3561               echo = (icmp_echo_header_t *) (icmp + 1);
3562               port = echo->identifier;
3563               break;
3564             case NAT_PROTOCOL_UDP:
3565             case NAT_PROTOCOL_TCP:
3566               port = ((tcp_udp_header_t *) l4_header)->src_port;
3567               break;
3568             default:
3569               next_worker_index = vlib_get_thread_index ();
3570               goto done;
3571             }
3572         }
3573     }
3574
3575   /* try static mappings with port */
3576   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3577     {
3578       init_nat_k (&kv, ip->dst_address, proto, 0, port);
3579       if (!clib_bihash_search_8_8
3580           (&sm->static_mapping_by_external, &kv, &value))
3581         {
3582           m = pool_elt_at_index (sm->static_mappings, value.value);
3583           if (!is_lb_static_mapping (m))
3584             {
3585               next_worker_index = m->workers[0];
3586               goto done;
3587             }
3588
3589           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3590             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3591
3592           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3593             next_worker_index =
3594               m->workers[hash & (_vec_len (m->workers) - 1)];
3595           else
3596             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3597           goto done;
3598         }
3599     }
3600
3601   /* worker by outside port */
3602   next_worker_index = sm->first_worker_index;
3603   next_worker_index +=
3604     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3605
3606 done:
3607   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3608                           clib_net_to_host_u32 (ip->src_address.as_u32),
3609                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3610   return next_worker_index;
3611 }
3612
3613 void
3614 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3615                 ip4_address_t * out_addr, u16 out_port,
3616                 ip4_address_t * eh_addr, u16 eh_port,
3617                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3618                 u32 fib_index, u16 flags, u32 thread_index)
3619 {
3620   snat_main_t *sm = &snat_main;
3621   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3622   snat_user_t *u;
3623   snat_session_t *s;
3624   clib_bihash_kv_8_8_t kv;
3625   vlib_main_t *vm = vlib_get_main ();
3626   f64 now = vlib_time_now (vm);
3627   nat_outside_fib_t *outside_fib;
3628   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3629   fib_prefix_t pfx = {
3630     .fp_proto = FIB_PROTOCOL_IP4,
3631     .fp_len = 32,
3632     .fp_addr = {
3633                 .ip4.as_u32 = eh_addr->as_u32,
3634                 },
3635   };
3636
3637   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3638     {
3639       if (nat_set_outside_address_and_port
3640           (sm->addresses, thread_index, *out_addr, out_port, proto))
3641         return;
3642     }
3643
3644   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3645   if (!u)
3646     return;
3647
3648   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3649   if (!s)
3650     return;
3651
3652   if (sm->endpoint_dependent)
3653     {
3654       nat_ed_lru_insert (tsm, s, now, nat_proto_to_ip_proto (proto));
3655     }
3656
3657   s->out2in.addr.as_u32 = out_addr->as_u32;
3658   s->out2in.port = out_port;
3659   s->nat_proto = proto;
3660   s->last_heard = now;
3661   s->flags = flags;
3662   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3663   s->ext_host_port = eh_port;
3664   user_session_increment (sm, u, snat_is_session_static (s));
3665   switch (vec_len (sm->outside_fibs))
3666     {
3667     case 0:
3668       s->out2in.fib_index = sm->outside_fib_index;
3669       break;
3670     case 1:
3671       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3672       break;
3673     default:
3674       /* *INDENT-OFF* */
3675       vec_foreach (outside_fib, sm->outside_fibs)
3676         {
3677           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3678           if (FIB_NODE_INDEX_INVALID != fei)
3679             {
3680               if (fib_entry_get_resolving_interface (fei) != ~0)
3681                 {
3682                   s->out2in.fib_index = outside_fib->fib_index;
3683                   break;
3684                 }
3685             }
3686         }
3687       /* *INDENT-ON* */
3688       break;
3689     }
3690   init_nat_o2i_kv (&kv, s, s - tsm->sessions);
3691   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3692     nat_elog_warn ("out2in key add failed");
3693
3694   s->in2out.addr.as_u32 = in_addr->as_u32;
3695   s->in2out.port = in_port;
3696   s->in2out.fib_index = fib_index;
3697   init_nat_i2o_kv (&kv, s, s - tsm->sessions);
3698   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3699     nat_elog_warn ("in2out key add failed");
3700 }
3701
3702 void
3703 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3704                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3705                 u32 ti)
3706 {
3707   snat_main_t *sm = &snat_main;
3708   clib_bihash_kv_8_8_t kv, value;
3709   u32 thread_index;
3710   snat_session_t *s;
3711   snat_main_per_thread_data_t *tsm;
3712
3713   if (sm->num_workers > 1)
3714     thread_index =
3715       sm->first_worker_index +
3716       (sm->workers[(clib_net_to_host_u16 (out_port) -
3717                     1024) / sm->port_per_thread]);
3718   else
3719     thread_index = sm->num_workers;
3720   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3721
3722   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3723   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3724     return;
3725
3726   s = pool_elt_at_index (tsm->sessions, value.value);
3727   nat_free_session_data (sm, s, thread_index, 1);
3728   nat44_delete_session (sm, s, thread_index);
3729 }
3730
3731 void
3732 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3733                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3734                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3735 {
3736   snat_main_t *sm = &snat_main;
3737   clib_bihash_kv_8_8_t kv, value;
3738   snat_session_t *s;
3739   snat_main_per_thread_data_t *tsm;
3740
3741   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3742
3743   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3744   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3745     return;
3746
3747   s = pool_elt_at_index (tsm->sessions, value.value);
3748   s->total_pkts = total_pkts;
3749   s->total_bytes = total_bytes;
3750 }
3751
3752 void
3753 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3754                    ip4_address_t * out_addr, u16 out_port,
3755                    ip4_address_t * eh_addr, u16 eh_port,
3756                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3757                    u32 fib_index, u16 flags, u32 thread_index)
3758 {
3759   snat_main_t *sm = &snat_main;
3760   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3761   snat_session_t *s;
3762   clib_bihash_kv_16_8_t kv;
3763   vlib_main_t *vm = vlib_get_main ();
3764   f64 now = vlib_time_now (vm);
3765   nat_outside_fib_t *outside_fib;
3766   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3767   fib_prefix_t pfx = {
3768     .fp_proto = FIB_PROTOCOL_IP4,
3769     .fp_len = 32,
3770     .fp_addr = {
3771                 .ip4.as_u32 = eh_addr->as_u32,
3772                 },
3773   };
3774
3775
3776   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3777     {
3778       if (nat_set_outside_address_and_port
3779           (sm->addresses, thread_index, *out_addr, out_port, proto))
3780         return;
3781     }
3782
3783   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3784     {
3785       if (nat_set_outside_address_and_port
3786           (sm->addresses, thread_index, *ehn_addr, ehn_port, proto))
3787         return;
3788     }
3789
3790   s = nat_ed_session_alloc (sm, thread_index, now, proto);
3791   if (!s)
3792     return;
3793
3794   s->last_heard = now;
3795   s->flags = flags;
3796   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3797   s->ext_host_nat_port = s->ext_host_port = eh_port;
3798   if (is_twice_nat_session (s))
3799     {
3800       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3801       s->ext_host_nat_port = ehn_port;
3802     }
3803   switch (vec_len (sm->outside_fibs))
3804     {
3805     case 0:
3806       s->out2in.fib_index = sm->outside_fib_index;
3807       break;
3808     case 1:
3809       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3810       break;
3811     default:
3812       /* *INDENT-OFF* */
3813       vec_foreach (outside_fib, sm->outside_fibs)
3814         {
3815           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3816           if (FIB_NODE_INDEX_INVALID != fei)
3817             {
3818               if (fib_entry_get_resolving_interface (fei) != ~0)
3819                 {
3820                   s->out2in.fib_index = outside_fib->fib_index;
3821                   break;
3822                 }
3823             }
3824         }
3825       /* *INDENT-ON* */
3826       break;
3827     }
3828   s->nat_proto = proto;
3829   s->out2in.addr.as_u32 = out_addr->as_u32;
3830   s->out2in.port = out_port;
3831
3832   s->in2out.addr.as_u32 = in_addr->as_u32;
3833   s->in2out.port = in_port;
3834   s->in2out.fib_index = fib_index;
3835
3836   init_ed_kv (&kv, *in_addr, in_port, s->ext_host_nat_addr,
3837               s->ext_host_nat_port, fib_index, nat_proto_to_ip_proto (proto),
3838               thread_index, s - tsm->sessions);
3839   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3840     nat_elog_warn ("in2out key add failed");
3841
3842   init_ed_kv (&kv, *out_addr, out_port, *eh_addr, eh_port,
3843               s->out2in.fib_index, nat_proto_to_ip_proto (proto),
3844               thread_index, s - tsm->sessions);
3845   if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 1))
3846     nat_elog_warn ("out2in key add failed");
3847 }
3848
3849 void
3850 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3851                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3852                    u32 fib_index, u32 ti)
3853 {
3854   snat_main_t *sm = &snat_main;
3855   clib_bihash_kv_16_8_t kv, value;
3856   u32 thread_index;
3857   snat_session_t *s;
3858   snat_main_per_thread_data_t *tsm;
3859
3860   if (sm->num_workers > 1)
3861     thread_index =
3862       sm->first_worker_index +
3863       (sm->workers[(clib_net_to_host_u16 (out_port) -
3864                     1024) / sm->port_per_thread]);
3865   else
3866     thread_index = sm->num_workers;
3867   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3868
3869   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3870   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3871     return;
3872
3873   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3874   nat_free_session_data (sm, s, thread_index, 1);
3875   nat44_delete_session (sm, s, thread_index);
3876 }
3877
3878 void
3879 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3880                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3881                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3882                    u32 thread_index)
3883 {
3884   snat_main_t *sm = &snat_main;
3885   clib_bihash_kv_16_8_t kv, value;
3886   snat_session_t *s;
3887   snat_main_per_thread_data_t *tsm;
3888
3889   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3890
3891   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3892   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3893     return;
3894
3895   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3896   s->total_pkts = total_pkts;
3897   s->total_bytes = total_bytes;
3898 }
3899
3900 static u32
3901 nat_calc_bihash_buckets (u32 n_elts)
3902 {
3903   return 1 << (max_log2 (n_elts >> 1) + 1);
3904 }
3905
3906 static u32
3907 nat_calc_bihash_memory (u32 n_buckets, uword kv_size)
3908 {
3909   return n_buckets * (8 + kv_size * 4);
3910 }
3911
3912 u32
3913 nat44_get_max_session_limit ()
3914 {
3915   snat_main_t *sm = &snat_main;
3916   u32 max_limit = 0, len = 0;
3917
3918   for (; len < vec_len (sm->max_translations_per_fib); len++)
3919     {
3920       if (max_limit < sm->max_translations_per_fib[len])
3921         max_limit = sm->max_translations_per_fib[len];
3922     }
3923   return max_limit;
3924 }
3925
3926 int
3927 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3928 {
3929   snat_main_t *sm = &snat_main;
3930   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3931   u32 len = vec_len (sm->max_translations_per_fib);
3932
3933   if (len <= fib_index)
3934     {
3935       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3936
3937       for (; len < vec_len (sm->max_translations_per_fib); len++)
3938         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3939     }
3940
3941   sm->max_translations_per_fib[fib_index] = session_limit;
3942   return 0;
3943 }
3944
3945 int
3946 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3947 {
3948   snat_main_t *sm = &snat_main;
3949
3950   if (nat44_set_session_limit (session_limit, vrf_id))
3951     return 1;
3952   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3953
3954   sm->translation_buckets =
3955     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3956
3957   if (!sm->translation_memory_size_set)
3958     {
3959       sm->translation_memory_size =
3960         nat_calc_bihash_memory (sm->translation_buckets,
3961                                 sizeof (clib_bihash_16_8_t));
3962     }
3963
3964   nat44_sessions_clear ();
3965   return 0;
3966 }
3967
3968 void
3969 nat44_db_init (snat_main_per_thread_data_t * tsm)
3970 {
3971   snat_main_t *sm = &snat_main;
3972
3973   pool_alloc (tsm->sessions, sm->max_translations_per_thread);
3974   pool_alloc (tsm->lru_pool, sm->max_translations_per_thread);
3975
3976   dlist_elt_t *head;
3977
3978   pool_get (tsm->lru_pool, head);
3979   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3980   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3981
3982   pool_get (tsm->lru_pool, head);
3983   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3984   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3985
3986   pool_get (tsm->lru_pool, head);
3987   tsm->udp_lru_head_index = head - tsm->lru_pool;
3988   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3989
3990   pool_get (tsm->lru_pool, head);
3991   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3992   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3993
3994   pool_get (tsm->lru_pool, head);
3995   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3996   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3997
3998   if (sm->endpoint_dependent)
3999     {
4000       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
4001                              sm->translation_buckets,
4002                              sm->translation_memory_size);
4003       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
4004                                           format_ed_session_kvp);
4005
4006     }
4007   else
4008     {
4009       clib_bihash_init_8_8 (&tsm->in2out, "in2out",
4010                             sm->translation_buckets,
4011                             sm->translation_memory_size);
4012       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
4013       clib_bihash_init_8_8 (&tsm->out2in, "out2in",
4014                             sm->translation_buckets,
4015                             sm->translation_memory_size);
4016       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
4017     }
4018
4019   // TODO: resolve static mappings (put only to !ED)
4020   pool_alloc (tsm->list_pool, sm->max_translations_per_thread);
4021   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets,
4022                         sm->user_memory_size);
4023   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
4024 }
4025
4026 void
4027 nat44_db_free (snat_main_per_thread_data_t * tsm)
4028 {
4029   snat_main_t *sm = &snat_main;
4030
4031   pool_free (tsm->sessions);
4032   pool_free (tsm->lru_pool);
4033
4034   if (sm->endpoint_dependent)
4035     {
4036       clib_bihash_free_16_8 (&tsm->in2out_ed);
4037       vec_free (tsm->per_vrf_sessions_vec);
4038     }
4039   else
4040     {
4041       clib_bihash_free_8_8 (&tsm->in2out);
4042       clib_bihash_free_8_8 (&tsm->out2in);
4043     }
4044
4045   // TODO: resolve static mappings (put only to !ED)
4046   pool_free (tsm->users);
4047   pool_free (tsm->list_pool);
4048   clib_bihash_free_8_8 (&tsm->user_hash);
4049 }
4050
4051 void
4052 nat44_sessions_clear ()
4053 {
4054   snat_main_t *sm = &snat_main;
4055   snat_main_per_thread_data_t *tsm;
4056
4057   if (sm->endpoint_dependent)
4058     {
4059       clib_bihash_free_16_8 (&sm->out2in_ed);
4060       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4061                              clib_max (1, sm->num_workers) *
4062                              sm->translation_buckets,
4063                              clib_max (1, sm->num_workers) *
4064                              sm->translation_memory_size);
4065       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4066                                           format_ed_session_kvp);
4067     }
4068
4069   /* *INDENT-OFF* */
4070   vec_foreach (tsm, sm->per_thread_data)
4071     {
4072       u32 ti;
4073
4074       nat44_db_free (tsm);
4075       nat44_db_init (tsm);
4076
4077       ti = tsm->snat_thread_index;
4078       vlib_set_simple_counter (&sm->total_users, ti, 0, 0);
4079       vlib_set_simple_counter (&sm->total_sessions, ti, 0, 0);
4080     }
4081   /* *INDENT-ON* */
4082 }
4083
4084 static clib_error_t *
4085 snat_config (vlib_main_t * vm, unformat_input_t * input)
4086 {
4087   snat_main_t *sm = &snat_main;
4088   snat_main_per_thread_data_t *tsm;
4089
4090   u32 static_mapping_buckets = 1024;
4091   uword static_mapping_memory_size = 64 << 20;
4092
4093   u32 nat64_bib_buckets = 1024;
4094   u32 nat64_bib_memory_size = 128 << 20;
4095
4096   u32 nat64_st_buckets = 2048;
4097   uword nat64_st_memory_size = 256 << 20;
4098
4099   u32 max_users_per_thread = 0;
4100   u32 user_memory_size = 0;
4101   u32 max_translations_per_thread = 0;
4102   u32 translation_memory_size = 0;
4103
4104   u32 max_translations_per_user = ~0;
4105
4106   u32 outside_vrf_id = 0;
4107   u32 outside_ip6_vrf_id = 0;
4108   u32 inside_vrf_id = 0;
4109   u8 static_mapping_only = 0;
4110   u8 static_mapping_connection_tracking = 0;
4111
4112   u32 udp_timeout = SNAT_UDP_TIMEOUT;
4113   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
4114   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
4115   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
4116
4117   sm->out2in_dpo = 0;
4118   sm->endpoint_dependent = 0;
4119
4120   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
4121     {
4122       if (unformat
4123           (input, "max translations per thread %d",
4124            &max_translations_per_thread))
4125         ;
4126       else if (unformat (input, "udp timeout %d", &udp_timeout))
4127         ;
4128       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
4129         ;
4130       else if (unformat (input, "tcp transitory timeout %d",
4131                          &tcp_transitory_timeout));
4132       else if (unformat (input, "tcp established timeout %d",
4133                          &tcp_established_timeout));
4134       else if (unformat (input, "translation hash memory %d",
4135                          &translation_memory_size));
4136       else
4137         if (unformat
4138             (input, "max users per thread %d", &max_users_per_thread))
4139         ;
4140       else if (unformat (input, "user hash memory %d", &user_memory_size))
4141         ;
4142       else if (unformat (input, "max translations per user %d",
4143                          &max_translations_per_user))
4144         ;
4145       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
4146         ;
4147       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
4148         ;
4149       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
4150         ;
4151       else if (unformat (input, "static mapping only"))
4152         {
4153           static_mapping_only = 1;
4154           if (unformat (input, "connection tracking"))
4155             static_mapping_connection_tracking = 1;
4156         }
4157       else if (unformat (input, "nat64 bib hash buckets %d",
4158                          &nat64_bib_buckets))
4159         ;
4160       else if (unformat (input, "nat64 bib hash memory %d",
4161                          &nat64_bib_memory_size))
4162         ;
4163       else
4164         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
4165         ;
4166       else if (unformat (input, "nat64 st hash memory %d",
4167                          &nat64_st_memory_size))
4168         ;
4169       else if (unformat (input, "out2in dpo"))
4170         sm->out2in_dpo = 1;
4171       else if (unformat (input, "endpoint-dependent"))
4172         sm->endpoint_dependent = 1;
4173       else
4174         return clib_error_return (0, "unknown input '%U'",
4175                                   format_unformat_error, input);
4176     }
4177
4178   if (static_mapping_only && (sm->endpoint_dependent))
4179     return clib_error_return (0,
4180                               "static mapping only mode available only for simple nat");
4181
4182   if (sm->out2in_dpo && (sm->endpoint_dependent))
4183     return clib_error_return (0,
4184                               "out2in dpo mode available only for simple nat");
4185   if (sm->endpoint_dependent && max_users_per_thread > 0)
4186     {
4187       return clib_error_return (0,
4188                                 "setting 'max users' in endpoint-dependent mode is not supported");
4189     }
4190
4191   if (sm->endpoint_dependent && max_translations_per_user != ~0)
4192     {
4193       return clib_error_return (0,
4194                                 "setting 'max translations per user' in endpoint-dependent mode is not supported");
4195     }
4196
4197   /* optionally configurable timeouts for testing purposes */
4198   sm->udp_timeout = udp_timeout;
4199   sm->tcp_transitory_timeout = tcp_transitory_timeout;
4200   sm->tcp_established_timeout = tcp_established_timeout;
4201   sm->icmp_timeout = icmp_timeout;
4202
4203   if (0 == max_users_per_thread)
4204     {
4205       max_users_per_thread = 1024;
4206     }
4207   sm->max_users_per_thread = max_users_per_thread;
4208   sm->user_buckets = nat_calc_bihash_buckets (sm->max_users_per_thread);
4209
4210   if (0 == max_translations_per_thread)
4211     {
4212       // default value based on legacy setting of load factor 10 * default
4213       // translation buckets 1024
4214       max_translations_per_thread = 10 * 1024;
4215     }
4216   sm->translation_memory_size_set = translation_memory_size != 0;
4217
4218   sm->max_translations_per_thread = max_translations_per_thread;
4219   sm->translation_buckets =
4220     nat_calc_bihash_buckets (sm->max_translations_per_thread);
4221   if (0 == translation_memory_size)
4222     {
4223       translation_memory_size =
4224         nat_calc_bihash_memory (sm->translation_buckets,
4225                                 sizeof (clib_bihash_16_8_t));
4226     }
4227   sm->translation_memory_size = translation_memory_size;
4228   if (0 == user_memory_size)
4229     {
4230       user_memory_size =
4231         nat_calc_bihash_memory (sm->max_users_per_thread,
4232                                 sizeof (clib_bihash_8_8_t));
4233     }
4234   sm->user_memory_size = user_memory_size;
4235   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
4236
4237   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4238     sm->max_translations_per_thread : max_translations_per_user;
4239
4240   sm->outside_vrf_id = outside_vrf_id;
4241   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4242                                                              outside_vrf_id,
4243                                                              nat_fib_src_hi);
4244   sm->inside_vrf_id = inside_vrf_id;
4245   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4246                                                             inside_vrf_id,
4247                                                             nat_fib_src_hi);
4248   sm->static_mapping_only = static_mapping_only;
4249   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4250
4251   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4252                   nat64_st_memory_size);
4253
4254   if (sm->endpoint_dependent)
4255     {
4256       sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4257       sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4258
4259       sm->in2out_node_index = nat44_ed_in2out_node.index;
4260       sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4261       sm->out2in_node_index = nat44_ed_out2in_node.index;
4262
4263       sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4264       sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4265       nat_affinity_init (vm);
4266       nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4267                    nat_ha_sref_ed_cb);
4268       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4269                              clib_max (1, sm->num_workers) *
4270                              sm->translation_buckets,
4271                              clib_max (1, sm->num_workers) *
4272                              sm->translation_memory_size);
4273       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4274                                           format_ed_session_kvp);
4275     }
4276   else
4277     {
4278       sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4279       sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4280
4281       sm->in2out_node_index = snat_in2out_node.index;
4282       sm->in2out_output_node_index = snat_in2out_output_node.index;
4283       sm->out2in_node_index = snat_out2in_node.index;
4284
4285       sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4286       sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4287       nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4288     }
4289   if (!static_mapping_only ||
4290       (static_mapping_only && static_mapping_connection_tracking))
4291     {
4292           /* *INDENT-OFF* */
4293           vec_foreach (tsm, sm->per_thread_data)
4294             {
4295               nat44_db_init (tsm);
4296             }
4297           /* *INDENT-ON* */
4298     }
4299   else
4300     {
4301       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4302       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4303     }
4304   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4305                         "static_mapping_by_local", static_mapping_buckets,
4306                         static_mapping_memory_size);
4307   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4308                                      format_static_mapping_kvp);
4309
4310   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4311                         "static_mapping_by_external",
4312                         static_mapping_buckets, static_mapping_memory_size);
4313   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4314                                      format_static_mapping_kvp);
4315
4316   return 0;
4317 }
4318
4319 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4320
4321 static void
4322 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4323                                  uword opaque,
4324                                  u32 sw_if_index,
4325                                  ip4_address_t * address,
4326                                  u32 address_length,
4327                                  u32 if_address_index, u32 is_delete)
4328 {
4329   snat_main_t *sm = &snat_main;
4330   snat_static_map_resolve_t *rp;
4331   snat_static_mapping_t *m;
4332   clib_bihash_kv_8_8_t kv, value;
4333   int i, rv;
4334   ip4_address_t l_addr;
4335
4336   for (i = 0; i < vec_len (sm->to_resolve); i++)
4337     {
4338       rp = sm->to_resolve + i;
4339       if (rp->addr_only == 0)
4340         continue;
4341       if (rp->sw_if_index == sw_if_index)
4342         goto match;
4343     }
4344
4345   return;
4346
4347 match:
4348   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
4349               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
4350   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4351     m = 0;
4352   else
4353     m = pool_elt_at_index (sm->static_mappings, value.value);
4354
4355   if (!is_delete)
4356     {
4357       /* Don't trip over lease renewal, static config */
4358       if (m)
4359         return;
4360     }
4361   else
4362     {
4363       if (!m)
4364         return;
4365     }
4366
4367   /* Indetity mapping? */
4368   if (rp->l_addr.as_u32 == 0)
4369     l_addr.as_u32 = address[0].as_u32;
4370   else
4371     l_addr.as_u32 = rp->l_addr.as_u32;
4372   /* Add the static mapping */
4373   rv = snat_add_static_mapping (l_addr,
4374                                 address[0],
4375                                 rp->l_port,
4376                                 rp->e_port,
4377                                 rp->vrf_id,
4378                                 rp->addr_only, ~0 /* sw_if_index */ ,
4379                                 rp->proto, !is_delete, rp->twice_nat,
4380                                 rp->out2in_only, rp->tag, rp->identity_nat,
4381                                 rp->pool_addr, rp->exact);
4382   if (rv)
4383     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4384 }
4385
4386 static void
4387 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4388                                        uword opaque,
4389                                        u32 sw_if_index,
4390                                        ip4_address_t * address,
4391                                        u32 address_length,
4392                                        u32 if_address_index, u32 is_delete)
4393 {
4394   snat_main_t *sm = &snat_main;
4395   snat_static_map_resolve_t *rp;
4396   ip4_address_t l_addr;
4397   int i, j;
4398   int rv;
4399   u8 twice_nat = 0;
4400   snat_address_t *addresses = sm->addresses;
4401
4402   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4403     {
4404       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4405         goto match;
4406     }
4407
4408   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4409     {
4410       twice_nat = 1;
4411       addresses = sm->twice_nat_addresses;
4412       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4413         goto match;
4414     }
4415
4416   return;
4417
4418 match:
4419   if (!is_delete)
4420     {
4421       /* Don't trip over lease renewal, static config */
4422       for (j = 0; j < vec_len (addresses); j++)
4423         if (addresses[j].addr.as_u32 == address->as_u32)
4424           return;
4425
4426       (void) snat_add_address (sm, address, ~0, twice_nat);
4427       /* Scan static map resolution vector */
4428       for (j = 0; j < vec_len (sm->to_resolve); j++)
4429         {
4430           rp = sm->to_resolve + j;
4431           if (rp->addr_only)
4432             continue;
4433           /* On this interface? */
4434           if (rp->sw_if_index == sw_if_index)
4435             {
4436               /* Indetity mapping? */
4437               if (rp->l_addr.as_u32 == 0)
4438                 l_addr.as_u32 = address[0].as_u32;
4439               else
4440                 l_addr.as_u32 = rp->l_addr.as_u32;
4441               /* Add the static mapping */
4442               rv = snat_add_static_mapping (l_addr,
4443                                             address[0],
4444                                             rp->l_port,
4445                                             rp->e_port,
4446                                             rp->vrf_id,
4447                                             rp->addr_only,
4448                                             ~0 /* sw_if_index */ ,
4449                                             rp->proto,
4450                                             rp->is_add, rp->twice_nat,
4451                                             rp->out2in_only, rp->tag,
4452                                             rp->identity_nat,
4453                                             rp->pool_addr, rp->exact);
4454               if (rv)
4455                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4456                                     "i4", rv);
4457             }
4458         }
4459       return;
4460     }
4461   else
4462     {
4463       (void) snat_del_address (sm, address[0], 1, twice_nat);
4464       return;
4465     }
4466 }
4467
4468
4469 int
4470 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4471                             u8 twice_nat)
4472 {
4473   ip4_main_t *ip4_main = sm->ip4_main;
4474   ip4_address_t *first_int_addr;
4475   snat_static_map_resolve_t *rp;
4476   u32 *indices_to_delete = 0;
4477   int i, j;
4478   u32 *auto_add_sw_if_indices =
4479     twice_nat ? sm->
4480     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4481
4482   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4483     );
4484
4485   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4486     {
4487       if (auto_add_sw_if_indices[i] == sw_if_index)
4488         {
4489           if (is_del)
4490             {
4491               /* if have address remove it */
4492               if (first_int_addr)
4493                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4494               else
4495                 {
4496                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4497                     {
4498                       rp = sm->to_resolve + j;
4499                       if (rp->sw_if_index == sw_if_index)
4500                         vec_add1 (indices_to_delete, j);
4501                     }
4502                   if (vec_len (indices_to_delete))
4503                     {
4504                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4505                         vec_del1 (sm->to_resolve, j);
4506                       vec_free (indices_to_delete);
4507                     }
4508                 }
4509               if (twice_nat)
4510                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4511               else
4512                 vec_del1 (sm->auto_add_sw_if_indices, i);
4513             }
4514           else
4515             return VNET_API_ERROR_VALUE_EXIST;
4516
4517           return 0;
4518         }
4519     }
4520
4521   if (is_del)
4522     return VNET_API_ERROR_NO_SUCH_ENTRY;
4523
4524   /* add to the auto-address list */
4525   if (twice_nat)
4526     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4527   else
4528     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4529
4530   /* If the address is already bound - or static - add it now */
4531   if (first_int_addr)
4532     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4533
4534   return 0;
4535 }
4536
4537 int
4538 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4539                    nat_protocol_t proto, u32 vrf_id, int is_in)
4540 {
4541   snat_main_per_thread_data_t *tsm;
4542   clib_bihash_kv_8_8_t kv, value;
4543   ip4_header_t ip;
4544   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4545   snat_session_t *s;
4546   clib_bihash_8_8_t *t;
4547
4548   if (sm->endpoint_dependent)
4549     return VNET_API_ERROR_UNSUPPORTED;
4550
4551   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4552   if (sm->num_workers > 1)
4553     tsm =
4554       vec_elt_at_index (sm->per_thread_data,
4555                         sm->worker_in2out_cb (&ip, fib_index, 0));
4556   else
4557     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4558
4559   init_nat_k (&kv, *addr, port, fib_index, proto);
4560   t = is_in ? &tsm->in2out : &tsm->out2in;
4561   if (!clib_bihash_search_8_8 (t, &kv, &value))
4562     {
4563       if (pool_is_free_index (tsm->sessions, value.value))
4564         return VNET_API_ERROR_UNSPECIFIED;
4565
4566       s = pool_elt_at_index (tsm->sessions, value.value);
4567       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4568       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4569       return 0;
4570     }
4571
4572   return VNET_API_ERROR_NO_SUCH_ENTRY;
4573 }
4574
4575 int
4576 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4577                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4578                       u32 vrf_id, int is_in)
4579 {
4580   ip4_header_t ip;
4581   clib_bihash_16_8_t *t;
4582   clib_bihash_kv_16_8_t kv, value;
4583   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4584   snat_session_t *s;
4585   snat_main_per_thread_data_t *tsm;
4586
4587   if (!sm->endpoint_dependent)
4588     return VNET_API_ERROR_FEATURE_DISABLED;
4589
4590   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4591   if (sm->num_workers > 1)
4592     tsm =
4593       vec_elt_at_index (sm->per_thread_data,
4594                         sm->worker_in2out_cb (&ip, fib_index, 0));
4595   else
4596     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4597
4598   t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
4599   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
4600   if (clib_bihash_search_16_8 (t, &kv, &value))
4601     {
4602       return VNET_API_ERROR_NO_SUCH_ENTRY;
4603     }
4604
4605   if (pool_is_free_index (tsm->sessions, value.value))
4606     return VNET_API_ERROR_UNSPECIFIED;
4607   s = pool_elt_at_index (tsm->sessions, value.value);
4608   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4609   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
4610   return 0;
4611 }
4612
4613 void
4614 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4615 {
4616   snat_main_t *sm = &snat_main;
4617
4618   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4619   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4620   sm->psid = psid;
4621   sm->psid_offset = psid_offset;
4622   sm->psid_length = psid_length;
4623 }
4624
4625 void
4626 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4627 {
4628   snat_main_t *sm = &snat_main;
4629
4630   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4631   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4632   sm->start_port = start_port;
4633   sm->end_port = end_port;
4634 }
4635
4636 void
4637 nat_set_alloc_addr_and_port_default (void)
4638 {
4639   snat_main_t *sm = &snat_main;
4640
4641   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4642   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4643 }
4644
4645 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4646                                  vlib_node_runtime_t * node,
4647                                  vlib_frame_t * frame)
4648 {
4649   return 0;
4650 }
4651
4652 /* *INDENT-OFF* */
4653 VLIB_REGISTER_NODE (nat_default_node) = {
4654   .name = "nat-default",
4655   .vector_size = sizeof (u32),
4656   .format_trace = 0,
4657   .type = VLIB_NODE_TYPE_INTERNAL,
4658   .n_errors = 0,
4659   .n_next_nodes = NAT_N_NEXT,
4660   .next_nodes = {
4661     [NAT_NEXT_DROP] = "error-drop",
4662     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4663     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4664     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4665     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4666     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4667     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4668     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4669     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4670   },
4671 };
4672 /* *INDENT-ON* */
4673
4674 /*
4675  * fd.io coding-style-patch-verification: ON
4676  *
4677  * Local Variables:
4678  * eval: (c-set-style "gnu")
4679  * End:
4680  */