src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat64.h>
  26 #include <nat/nat_inlines.h>
  27 #include <nat/nat44/inlines.h>
  28 #include <nat/nat_affinity.h>
  29 #include <nat/nat_syslog.h>
  30 #include <nat/nat_ha.h>
  31 #include <vnet/fib/fib_table.h>
  32 #include <vnet/fib/ip4_fib.h>
  33 #include <vnet/ip/reass/ip4_sv_reass.h>
  34 #include <vppinfra/bihash_16_8.h>
  35 #include <nat/nat44/ed_inlines.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 fib_source_t nat_fib_src_hi;
  42 fib_source_t nat_fib_src_low;
  43
  44 /* *INDENT-OFF* */
  45 /* Hook up input features */
  46 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat-pre-in2out",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-sv-reassembly-feature"),
  51 };
  52 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat-pre-out2in",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  56                                "ip4-dhcp-client-detect",
  57                                "ip4-sv-reassembly-feature"),
  58 };
  59 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-in2out-worker-handoff",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-out2in-worker-handoff",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-in2out",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  74 };
  75 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-out2in",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  79                                "ip4-dhcp-client-detect"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-classify",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  85 };
  86 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-ed-in2out",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  90 };
  91 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-ed-out2in",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  95                                "ip4-dhcp-client-detect"),
  96 };
  97 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-ed-classify",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-handoff-classify",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-in2out-fast",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 111 };
 112 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-out2in-fast",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 116                                "ip4-dhcp-client-detect"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-hairpin-dst",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ed-hairpin-dst",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 127 };
 128
 129 /* Hook up output features */
 130 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 131   .arc_name = "ip4-output",
 132   .node_name = "nat44-in2out-output",
 133   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 136   .arc_name = "ip4-output",
 137   .node_name = "nat44-in2out-output-worker-handoff",
 138   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 141   .arc_name = "ip4-output",
 142   .node_name = "nat44-hairpin-src",
 143   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 144 };
 145 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-ed-in2out-output",
 148   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 149   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 150 };
 151 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-ed-hairpin-src",
 154   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 155   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 156 };
 157
 158 /* Hook up ip4-local features */
 159 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 160 {
 161   .arc_name = "ip4-local",
 162   .node_name = "nat44-hairpinning",
 163   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 164 };
 165 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 166 {
 167   .arc_name = "ip4-local",
 168   .node_name = "nat44-ed-hairpinning",
 169   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 170 };
 171
 172
 173 VLIB_PLUGIN_REGISTER () = {
 174     .version = VPP_BUILD_VER,
 175     .description = "Network Address Translation (NAT)",
 176 };
 177 /* *INDENT-ON* */
 178
 179 void
 180 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 181                        u8 is_ha)
 182 {
 183   clib_bihash_kv_8_8_t kv;
 184   u8 proto;
 185   u16 r_port, l_port;
 186   ip4_address_t *l_addr, *r_addr;
 187   u32 fib_index = 0;
 188   clib_bihash_kv_16_8_t ed_kv;
 189   snat_main_per_thread_data_t *tsm =
 190     vec_elt_at_index (sm->per_thread_data, thread_index);
 191
 192   if (is_fwd_bypass_session (s))
 193     {
 194       if (snat_is_unk_proto_session (s))
 195         {
 196           init_ed_k (&ed_kv, s->in2out.addr, 0, s->ext_host_addr, 0, 0,
 197                      s->in2out.port);
 198         }
 199       else
 200         {
 201           l_port = s->in2out.port;
 202           r_port = s->ext_host_port;
 203           l_addr = &s->in2out.addr;
 204           r_addr = &s->ext_host_addr;
 205           proto = nat_proto_to_ip_proto (s->nat_proto);
 206           init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index,
 207                      proto);
 208         }
 209       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 210         nat_elog_warn ("in2out_ed key del failed");
 211       return;
 212     }
 213
 214   /* session lookup tables */
 215   if (is_ed_session (s))
 216     {
 217       if (is_affinity_sessions (s))
 218         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 219                              s->nat_proto, s->out2in.port);
 220       l_addr = &s->out2in.addr;
 221       r_addr = &s->ext_host_addr;
 222       fib_index = s->out2in.fib_index;
 223       if (snat_is_unk_proto_session (s))
 224         {
 225           proto = s->in2out.port;
 226           r_port = 0;
 227           l_port = 0;
 228         }
 229       else
 230         {
 231           proto = nat_proto_to_ip_proto (s->nat_proto);
 232           l_port = s->out2in.port;
 233           r_port = s->ext_host_port;
 234         }
 235       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 236       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 237         nat_elog_warn ("out2in_ed key del failed");
 238       l_addr = &s->in2out.addr;
 239       fib_index = s->in2out.fib_index;
 240       if (!snat_is_unk_proto_session (s))
 241         l_port = s->in2out.port;
 242       if (is_twice_nat_session (s))
 243         {
 244           r_addr = &s->ext_host_nat_addr;
 245           r_port = s->ext_host_nat_port;
 246         }
 247       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 248       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 249         nat_elog_warn ("in2out_ed key del failed");
 250
 251       if (!is_ha)
 252         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 253                                &s->in2out.addr, s->in2out.port,
 254                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 255                                &s->out2in.addr, s->out2in.port,
 256                                &s->ext_host_addr, s->ext_host_port,
 257                                s->nat_proto, is_twice_nat_session (s));
 258     }
 259   else
 260     {
 261       init_nat_i2o_k (&kv, s);
 262       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 263         nat_elog_warn ("in2out key del failed");
 264       init_nat_o2i_k (&kv, s);
 265       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 266         nat_elog_warn ("out2in key del failed");
 267
 268       if (!is_ha)
 269         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 270                                  &s->in2out.addr, s->in2out.port,
 271                                  &s->out2in.addr, s->out2in.port,
 272                                  s->nat_proto);
 273     }
 274
 275   if (snat_is_unk_proto_session (s))
 276     return;
 277
 278   if (!is_ha)
 279     {
 280       /* log NAT event */
 281       snat_ipfix_logging_nat44_ses_delete (thread_index,
 282                                            s->in2out.addr.as_u32,
 283                                            s->out2in.addr.as_u32,
 284                                            s->nat_proto,
 285                                            s->in2out.port,
 286                                            s->out2in.port,
 287                                            s->in2out.fib_index);
 288
 289       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 290                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 291                    thread_index);
 292     }
 293
 294   /* Twice NAT address and port for external host */
 295   if (is_twice_nat_session (s))
 296     {
 297       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 298                                           thread_index,
 299                                           &s->ext_host_nat_addr,
 300                                           s->ext_host_nat_port, s->nat_proto);
 301     }
 302
 303   if (snat_is_session_static (s))
 304     return;
 305
 306   snat_free_outside_address_and_port (sm->addresses, thread_index,
 307                                       &s->out2in.addr, s->out2in.port,
 308                                       s->nat_proto);
 309 }
 310
 311 int
 312 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
 313 {
 314   snat_main_t *sm = &snat_main;
 315   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
 316   u32 len = vec_len (sm->max_translations_per_fib);
 317
 318   if (len <= fib_index)
 319     {
 320       vec_validate (sm->max_translations_per_fib, fib_index + 1);
 321
 322       for (; len < vec_len (sm->max_translations_per_fib); len++)
 323         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
 324     }
 325
 326   sm->max_translations_per_fib[fib_index] = session_limit;
 327   return 0;
 328 }
 329
 330 void
 331 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 332                          u32 thread_index, u8 is_ha)
 333 {
 334   u8 proto;
 335   u16 r_port, l_port;
 336   ip4_address_t *l_addr, *r_addr;
 337   u32 fib_index;
 338   clib_bihash_kv_16_8_t ed_kv;
 339   snat_main_per_thread_data_t *tsm =
 340     vec_elt_at_index (sm->per_thread_data, thread_index);
 341
 342   if (is_fwd_bypass_session (s))
 343     {
 344       if (snat_is_unk_proto_session (s))
 345         {
 346           proto = s->in2out.port;
 347           r_port = 0;
 348           l_port = 0;
 349         }
 350       else
 351         {
 352           proto = nat_proto_to_ip_proto (s->nat_proto);
 353           l_port = s->in2out.port;
 354           r_port = s->ext_host_port;
 355         }
 356
 357       l_addr = &s->in2out.addr;
 358       r_addr = &s->ext_host_addr;
 359       fib_index = 0;
 360       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 361
 362       if (PREDICT_FALSE
 363           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 364         nat_elog_warn ("in2out_ed key del failed");
 365       return;
 366     }
 367
 368   /* session lookup tables */
 369   if (is_affinity_sessions (s))
 370     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 371                          s->nat_proto, s->out2in.port);
 372   l_addr = &s->out2in.addr;
 373   r_addr = &s->ext_host_addr;
 374   fib_index = s->out2in.fib_index;
 375   if (snat_is_unk_proto_session (s))
 376     {
 377       proto = s->in2out.port;
 378       r_port = 0;
 379       l_port = 0;
 380     }
 381   else
 382     {
 383       proto = nat_proto_to_ip_proto (s->nat_proto);
 384       l_port = s->out2in.port;
 385       r_port = s->ext_host_port;
 386     }
 387   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 388
 389   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0)))
 390     nat_elog_warn ("out2in_ed key del failed");
 391
 392   l_addr = &s->in2out.addr;
 393   fib_index = s->in2out.fib_index;
 394
 395   if (!snat_is_unk_proto_session (s))
 396     l_port = s->in2out.port;
 397
 398   if (is_twice_nat_session (s))
 399     {
 400       r_addr = &s->ext_host_nat_addr;
 401       r_port = s->ext_host_nat_port;
 402     }
 403   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 404
 405   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 406     nat_elog_warn ("in2out_ed key del failed");
 407
 408   if (!is_ha)
 409     {
 410       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 411                              &s->in2out.addr, s->in2out.port,
 412                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 413                              &s->out2in.addr, s->out2in.port,
 414                              &s->ext_host_addr, s->ext_host_port,
 415                              s->nat_proto, is_twice_nat_session (s));
 416     }
 417
 418   if (snat_is_unk_proto_session (s))
 419     return;
 420
 421   if (!is_ha)
 422     {
 423       snat_ipfix_logging_nat44_ses_delete (thread_index,
 424                                            s->in2out.addr.as_u32,
 425                                            s->out2in.addr.as_u32,
 426                                            s->nat_proto,
 427                                            s->in2out.port,
 428                                            s->out2in.port,
 429                                            s->in2out.fib_index);
 430       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 431                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 432                    thread_index);
 433     }
 434
 435   /* Twice NAT address and port for external host */
 436   if (is_twice_nat_session (s))
 437     {
 438       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 439                                           thread_index,
 440                                           &s->ext_host_nat_addr,
 441                                           s->ext_host_nat_port, s->nat_proto);
 442     }
 443
 444   if (snat_is_session_static (s))
 445     return;
 446
 447   snat_free_outside_address_and_port (sm->addresses, thread_index,
 448                                       &s->out2in.addr, s->out2in.port,
 449                                       s->nat_proto);
 450 }
 451
 452
 453 snat_user_t *
 454 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 455                         u32 thread_index)
 456 {
 457   snat_user_t *u = 0;
 458   snat_user_key_t user_key;
 459   clib_bihash_kv_8_8_t kv, value;
 460   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 461   dlist_elt_t *per_user_list_head_elt;
 462
 463   user_key.addr.as_u32 = addr->as_u32;
 464   user_key.fib_index = fib_index;
 465   kv.key = user_key.as_u64;
 466
 467   /* Ever heard of the "user" = src ip4 address before? */
 468   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 469     {
 470       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 471         {
 472           vlib_increment_simple_counter (&sm->user_limit_reached,
 473                                          thread_index, 0, 1);
 474           nat_elog_warn ("maximum user limit reached");
 475           return NULL;
 476         }
 477       /* no, make a new one */
 478       pool_get (tsm->users, u);
 479       clib_memset (u, 0, sizeof (*u));
 480
 481       u->addr.as_u32 = addr->as_u32;
 482       u->fib_index = fib_index;
 483
 484       pool_get (tsm->list_pool, per_user_list_head_elt);
 485
 486       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 487         tsm->list_pool;
 488
 489       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 490
 491       kv.value = u - tsm->users;
 492
 493       /* add user */
 494       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 495         {
 496           nat_elog_warn ("user_hash key add failed");
 497           nat44_delete_user_with_no_session (sm, u, thread_index);
 498           return NULL;
 499         }
 500
 501       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 502                                pool_elts (tsm->users));
 503     }
 504   else
 505     {
 506       u = pool_elt_at_index (tsm->users, value.value);
 507     }
 508
 509   return u;
 510 }
 511
 512 snat_session_t *
 513 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 514                               u32 thread_index, f64 now)
 515 {
 516   snat_session_t *s;
 517   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 518   u32 oldest_per_user_translation_list_index, session_index;
 519   dlist_elt_t *oldest_per_user_translation_list_elt;
 520   dlist_elt_t *per_user_translation_list_elt;
 521
 522   /* Over quota? Recycle the least recently used translation */
 523   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 524     {
 525       oldest_per_user_translation_list_index =
 526         clib_dlist_remove_head (tsm->list_pool,
 527                                 u->sessions_per_user_list_head_index);
 528
 529       ASSERT (oldest_per_user_translation_list_index != ~0);
 530
 531       /* Add it back to the end of the LRU list */
 532       clib_dlist_addtail (tsm->list_pool,
 533                           u->sessions_per_user_list_head_index,
 534                           oldest_per_user_translation_list_index);
 535       /* Get the list element */
 536       oldest_per_user_translation_list_elt =
 537         pool_elt_at_index (tsm->list_pool,
 538                            oldest_per_user_translation_list_index);
 539
 540       /* Get the session index from the list element */
 541       session_index = oldest_per_user_translation_list_elt->value;
 542
 543       /* Get the session */
 544       s = pool_elt_at_index (tsm->sessions, session_index);
 545       nat_free_session_data (sm, s, thread_index, 0);
 546       if (snat_is_session_static (s))
 547         u->nstaticsessions--;
 548       else
 549         u->nsessions--;
 550       s->flags = 0;
 551       s->total_bytes = 0;
 552       s->total_pkts = 0;
 553       s->state = 0;
 554       s->ext_host_addr.as_u32 = 0;
 555       s->ext_host_port = 0;
 556       s->ext_host_nat_addr.as_u32 = 0;
 557       s->ext_host_nat_port = 0;
 558     }
 559   else
 560     {
 561       pool_get (tsm->sessions, s);
 562       clib_memset (s, 0, sizeof (*s));
 563
 564       /* Create list elts */
 565       pool_get (tsm->list_pool, per_user_translation_list_elt);
 566       clib_dlist_init (tsm->list_pool,
 567                        per_user_translation_list_elt - tsm->list_pool);
 568
 569       per_user_translation_list_elt->value = s - tsm->sessions;
 570       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 571       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 572
 573       clib_dlist_addtail (tsm->list_pool,
 574                           s->per_user_list_head_index,
 575                           per_user_translation_list_elt - tsm->list_pool);
 576
 577       s->user_index = u - tsm->users;
 578       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 579                                pool_elts (tsm->sessions));
 580     }
 581
 582   s->ha_last_refreshed = now;
 583
 584   return s;
 585 }
 586
 587 void
 588 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 589                           int is_add)
 590 {
 591   fib_prefix_t prefix = {
 592     .fp_len = p_len,
 593     .fp_proto = FIB_PROTOCOL_IP4,
 594     .fp_addr = {
 595                 .ip4.as_u32 = addr->as_u32,
 596                 },
 597   };
 598   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 599
 600   if (is_add)
 601     fib_table_entry_update_one_path (fib_index,
 602                                      &prefix,
 603                                      nat_fib_src_low,
 604                                      (FIB_ENTRY_FLAG_CONNECTED |
 605                                       FIB_ENTRY_FLAG_LOCAL |
 606                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 607                                      DPO_PROTO_IP4,
 608                                      NULL,
 609                                      sw_if_index,
 610                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 611   else
 612     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 613 }
 614
 615 int
 616 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 617                   u8 twice_nat)
 618 {
 619   snat_address_t *ap;
 620   snat_interface_t *i;
 621   vlib_thread_main_t *tm = vlib_get_thread_main ();
 622
 623   if (twice_nat && !sm->endpoint_dependent)
 624     return VNET_API_ERROR_FEATURE_DISABLED;
 625
 626   /* Check if address already exists */
 627   /* *INDENT-OFF* */
 628   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 629     {
 630       if (ap->addr.as_u32 == addr->as_u32)
 631         return VNET_API_ERROR_VALUE_EXIST;
 632     }
 633   /* *INDENT-ON* */
 634
 635   if (twice_nat)
 636     vec_add2 (sm->twice_nat_addresses, ap, 1);
 637   else
 638     vec_add2 (sm->addresses, ap, 1);
 639
 640   ap->addr = *addr;
 641   if (vrf_id != ~0)
 642     ap->fib_index =
 643       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 644                                          nat_fib_src_low);
 645   else
 646     ap->fib_index = ~0;
 647 #define _(N, i, n, s) \
 648   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 649   ap->busy_##n##_ports = 0; \
 650   ap->busy_##n##_ports_per_thread = 0;\
 651   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 652   foreach_nat_protocol
 653 #undef _
 654     if (twice_nat)
 655     return 0;
 656
 657   /* Add external address to FIB */
 658   /* *INDENT-OFF* */
 659   pool_foreach (i, sm->interfaces,
 660   ({
 661     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 662       continue;
 663
 664     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 665     break;
 666   }));
 667   pool_foreach (i, sm->output_feature_interfaces,
 668   ({
 669     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 670       continue;
 671
 672     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 673     break;
 674   }));
 675   /* *INDENT-ON* */
 676
 677   return 0;
 678 }
 679
 680 static int
 681 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 682 {
 683   snat_static_mapping_t *m;
 684   /* *INDENT-OFF* */
 685   pool_foreach (m, sm->static_mappings,
 686   ({
 687       if (is_addr_only_static_mapping (m) ||
 688           is_out2in_only_static_mapping (m) ||
 689           is_identity_static_mapping (m))
 690         continue;
 691       if (m->external_addr.as_u32 == addr.as_u32)
 692         return 1;
 693   }));
 694   /* *INDENT-ON* */
 695
 696   return 0;
 697 }
 698
 699 static void
 700 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 701                                        ip4_address_t l_addr,
 702                                        u16 l_port,
 703                                        u32 sw_if_index,
 704                                        u16 e_port,
 705                                        u32 vrf_id,
 706                                        nat_protocol_t proto,
 707                                        int addr_only, int is_add, u8 * tag,
 708                                        int twice_nat, int out2in_only,
 709                                        int identity_nat)
 710 {
 711   snat_static_map_resolve_t *rp;
 712
 713   vec_add2 (sm->to_resolve, rp, 1);
 714   rp->l_addr.as_u32 = l_addr.as_u32;
 715   rp->l_port = l_port;
 716   rp->sw_if_index = sw_if_index;
 717   rp->e_port = e_port;
 718   rp->vrf_id = vrf_id;
 719   rp->proto = proto;
 720   rp->addr_only = addr_only;
 721   rp->is_add = is_add;
 722   rp->twice_nat = twice_nat;
 723   rp->out2in_only = out2in_only;
 724   rp->identity_nat = identity_nat;
 725   rp->tag = vec_dup (tag);
 726 }
 727
 728 static u32
 729 get_thread_idx_by_port (u16 e_port)
 730 {
 731   snat_main_t *sm = &snat_main;
 732   u32 thread_idx = sm->num_workers;
 733   if (sm->num_workers > 1)
 734     {
 735       thread_idx =
 736         sm->first_worker_index +
 737         sm->workers[(e_port - 1024) / sm->port_per_thread];
 738     }
 739   return thread_idx;
 740 }
 741
 742 void
 743 snat_static_mapping_del_sessions (snat_main_t * sm,
 744                                   snat_main_per_thread_data_t * tsm,
 745                                   snat_user_key_t u_key, int addr_only,
 746                                   ip4_address_t e_addr, u16 e_port)
 747 {
 748   clib_bihash_kv_8_8_t kv, value;
 749   kv.key = u_key.as_u64;
 750   u64 user_index;
 751   dlist_elt_t *head, *elt;
 752   snat_user_t *u;
 753   snat_session_t *s;
 754   u32 elt_index, head_index, ses_index;
 755   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 756     {
 757       user_index = value.value;
 758       u = pool_elt_at_index (tsm->users, user_index);
 759       if (u->nstaticsessions)
 760         {
 761           head_index = u->sessions_per_user_list_head_index;
 762           head = pool_elt_at_index (tsm->list_pool, head_index);
 763           elt_index = head->next;
 764           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 765           ses_index = elt->value;
 766           while (ses_index != ~0)
 767             {
 768               s = pool_elt_at_index (tsm->sessions, ses_index);
 769               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 770               ses_index = elt->value;
 771
 772               if (!addr_only)
 773                 {
 774                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 775                       (s->out2in.port != e_port))
 776                     continue;
 777                 }
 778
 779               if (is_lb_session (s))
 780                 continue;
 781
 782               if (!snat_is_session_static (s))
 783                 continue;
 784
 785               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 786               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 787
 788               if (!addr_only)
 789                 break;
 790             }
 791         }
 792     }
 793 }
 794
 795 void
 796 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 797                                      snat_main_per_thread_data_t * tsm,
 798                                      ip4_address_t l_addr,
 799                                      u16 l_port,
 800                                      u8 protocol,
 801                                      u32 fib_index, int addr_only,
 802                                      ip4_address_t e_addr, u16 e_port)
 803 {
 804   snat_session_t *s;
 805   u32 *indexes_to_free = NULL;
 806   /* *INDENT-OFF* */
 807   pool_foreach (s, tsm->sessions, {
 808     if (s->in2out.fib_index != fib_index ||
 809         s->in2out.addr.as_u32 != l_addr.as_u32)
 810       {
 811         continue;
 812       }
 813     if (!addr_only)
 814       {
 815         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 816             s->out2in.port != e_port ||
 817             s->in2out.port != l_port ||
 818             s->nat_proto != protocol)
 819           continue;
 820       }
 821
 822     if (is_lb_session (s))
 823       continue;
 824     if (!snat_is_session_static (s))
 825       continue;
 826     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 827     vec_add1 (indexes_to_free, s - tsm->sessions);
 828     if (!addr_only)
 829       break;
 830   });
 831   /* *INDENT-ON* */
 832   u32 *ses_index;
 833   vec_foreach (ses_index, indexes_to_free)
 834   {
 835     s = pool_elt_at_index (tsm->sessions, *ses_index);
 836     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 837   }
 838   vec_free (indexes_to_free);
 839 }
 840
 841 int
 842 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 843                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 844                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 845                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 846                          u8 identity_nat)
 847 {
 848   snat_main_t *sm = &snat_main;
 849   snat_static_mapping_t *m;
 850   clib_bihash_kv_8_8_t kv, value;
 851   snat_address_t *a = 0;
 852   u32 fib_index = ~0;
 853   snat_interface_t *interface;
 854   int i;
 855   snat_main_per_thread_data_t *tsm;
 856   snat_user_key_t u_key;
 857   snat_user_t *u;
 858   dlist_elt_t *head, *elt;
 859   u32 elt_index, head_index;
 860   u32 ses_index;
 861   u64 user_index;
 862   snat_session_t *s;
 863   snat_static_map_resolve_t *rp, *rp_match = 0;
 864   nat44_lb_addr_port_t *local;
 865   u32 find = ~0;
 866
 867   if (!sm->endpoint_dependent)
 868     {
 869       if (twice_nat || out2in_only)
 870         return VNET_API_ERROR_FEATURE_DISABLED;
 871     }
 872
 873   /* If the external address is a specific interface address */
 874   if (sw_if_index != ~0)
 875     {
 876       ip4_address_t *first_int_addr;
 877
 878       for (i = 0; i < vec_len (sm->to_resolve); i++)
 879         {
 880           rp = sm->to_resolve + i;
 881           if (rp->sw_if_index != sw_if_index ||
 882               rp->l_addr.as_u32 != l_addr.as_u32 ||
 883               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 884             continue;
 885
 886           if (!addr_only)
 887             {
 888               if ((rp->l_port != l_port && rp->e_port != e_port)
 889                   || rp->proto != proto)
 890                 continue;
 891             }
 892
 893           rp_match = rp;
 894           break;
 895         }
 896
 897       /* Might be already set... */
 898       first_int_addr = ip4_interface_first_address
 899         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 900
 901       if (is_add)
 902         {
 903           if (rp_match)
 904             return VNET_API_ERROR_VALUE_EXIST;
 905
 906           snat_add_static_mapping_when_resolved
 907             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 908              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 909
 910           /* DHCP resolution required? */
 911           if (first_int_addr == 0)
 912             {
 913               return 0;
 914             }
 915           else
 916             {
 917               e_addr.as_u32 = first_int_addr->as_u32;
 918               /* Identity mapping? */
 919               if (l_addr.as_u32 == 0)
 920                 l_addr.as_u32 = e_addr.as_u32;
 921             }
 922         }
 923       else
 924         {
 925           if (!rp_match)
 926             return VNET_API_ERROR_NO_SUCH_ENTRY;
 927
 928           vec_del1 (sm->to_resolve, i);
 929
 930           if (first_int_addr)
 931             {
 932               e_addr.as_u32 = first_int_addr->as_u32;
 933               /* Identity mapping? */
 934               if (l_addr.as_u32 == 0)
 935                 l_addr.as_u32 = e_addr.as_u32;
 936             }
 937           else
 938             return 0;
 939         }
 940     }
 941
 942   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 943   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 944     m = 0;
 945   else
 946     m = pool_elt_at_index (sm->static_mappings, value.value);
 947
 948   if (is_add)
 949     {
 950       if (m)
 951         {
 952           if (is_identity_static_mapping (m))
 953             {
 954               /* *INDENT-OFF* */
 955               pool_foreach (local, m->locals,
 956               ({
 957                 if (local->vrf_id == vrf_id)
 958                   return VNET_API_ERROR_VALUE_EXIST;
 959               }));
 960               /* *INDENT-ON* */
 961               pool_get (m->locals, local);
 962               local->vrf_id = vrf_id;
 963               local->fib_index =
 964                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 965                                                    nat_fib_src_low);
 966               init_nat_kv (&kv, m->local_addr, m->local_port,
 967                            local->fib_index, m->proto,
 968                            m - sm->static_mappings);
 969               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 970               return 0;
 971             }
 972           else
 973             return VNET_API_ERROR_VALUE_EXIST;
 974         }
 975
 976       if (twice_nat && addr_only)
 977         return VNET_API_ERROR_UNSUPPORTED;
 978
 979       /* Convert VRF id to FIB index */
 980       if (vrf_id != ~0)
 981         fib_index =
 982           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 983                                              nat_fib_src_low);
 984       /* If not specified use inside VRF id from SNAT plugin startup config */
 985       else
 986         {
 987           fib_index = sm->inside_fib_index;
 988           vrf_id = sm->inside_vrf_id;
 989           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 990         }
 991
 992       if (!(out2in_only || identity_nat))
 993         {
 994           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
 995                       addr_only ? 0 : proto);
 996           if (!clib_bihash_search_8_8
 997               (&sm->static_mapping_by_local, &kv, &value))
 998             return VNET_API_ERROR_VALUE_EXIST;
 999         }
1000
1001       /* Find external address in allocated addresses and reserve port for
1002          address and port pair mapping when dynamic translations enabled */
1003       if (!(addr_only || sm->static_mapping_only || out2in_only))
1004         {
1005           for (i = 0; i < vec_len (sm->addresses); i++)
1006             {
1007               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1008                 {
1009                   a = sm->addresses + i;
1010                   /* External port must be unused */
1011                   switch (proto)
1012                     {
1013 #define _(N, j, n, s) \
1014                     case NAT_PROTOCOL_##N: \
1015                       if (a->busy_##n##_port_refcounts[e_port]) \
1016                         return VNET_API_ERROR_INVALID_VALUE; \
1017                       ++a->busy_##n##_port_refcounts[e_port]; \
1018                       if (e_port > 1024) \
1019                         { \
1020                           a->busy_##n##_ports++; \
1021                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1022                         } \
1023                       break;
1024                       foreach_nat_protocol
1025 #undef _
1026                     default:
1027                       nat_elog_info ("unknown protocol");
1028                       return VNET_API_ERROR_INVALID_VALUE_2;
1029                     }
1030                   break;
1031                 }
1032             }
1033           /* External address must be allocated */
1034           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1035             {
1036               if (sw_if_index != ~0)
1037                 {
1038                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1039                     {
1040                       rp = sm->to_resolve + i;
1041                       if (rp->addr_only)
1042                         continue;
1043                       if (rp->sw_if_index != sw_if_index &&
1044                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1045                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1046                           rp->e_port != e_port && rp->proto != proto)
1047                         continue;
1048
1049                       vec_del1 (sm->to_resolve, i);
1050                       break;
1051                     }
1052                 }
1053               return VNET_API_ERROR_NO_SUCH_ENTRY;
1054             }
1055         }
1056
1057       pool_get (sm->static_mappings, m);
1058       clib_memset (m, 0, sizeof (*m));
1059       m->tag = vec_dup (tag);
1060       m->local_addr = l_addr;
1061       m->external_addr = e_addr;
1062       m->twice_nat = twice_nat;
1063       if (out2in_only)
1064         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1065       if (addr_only)
1066         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1067       if (identity_nat)
1068         {
1069           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1070           pool_get (m->locals, local);
1071           local->vrf_id = vrf_id;
1072           local->fib_index = fib_index;
1073         }
1074       else
1075         {
1076           m->vrf_id = vrf_id;
1077           m->fib_index = fib_index;
1078         }
1079       if (!addr_only)
1080         {
1081           m->local_port = l_port;
1082           m->external_port = e_port;
1083           m->proto = proto;
1084         }
1085
1086       if (sm->num_workers > 1)
1087         {
1088           ip4_header_t ip = {
1089             .src_address = m->local_addr,
1090           };
1091           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1092           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1093         }
1094       else
1095         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1096
1097       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
1098                    m - sm->static_mappings);
1099       if (!out2in_only)
1100         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1101
1102       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1103                    m - sm->static_mappings);
1104       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1105
1106       /* Delete dynamic sessions matching local address (+ local port) */
1107       if (!(sm->static_mapping_only))
1108         {
1109           u_key.addr = m->local_addr;
1110           u_key.fib_index = m->fib_index;
1111           kv.key = u_key.as_u64;
1112           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1113             {
1114               user_index = value.value;
1115               u = pool_elt_at_index (tsm->users, user_index);
1116               if (u->nsessions)
1117                 {
1118                   head_index = u->sessions_per_user_list_head_index;
1119                   head = pool_elt_at_index (tsm->list_pool, head_index);
1120                   elt_index = head->next;
1121                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1122                   ses_index = elt->value;
1123                   while (ses_index != ~0)
1124                     {
1125                       s = pool_elt_at_index (tsm->sessions, ses_index);
1126                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1127                       ses_index = elt->value;
1128
1129                       if (snat_is_session_static (s))
1130                         continue;
1131
1132                       if (!addr_only && s->in2out.port != m->local_port)
1133                         continue;
1134
1135                       nat_free_session_data (sm, s,
1136                                              tsm - sm->per_thread_data, 0);
1137                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1138
1139                       if (!addr_only && !sm->endpoint_dependent)
1140                         break;
1141                     }
1142                 }
1143             }
1144         }
1145     }
1146   else
1147     {
1148       if (!m)
1149         {
1150           if (sw_if_index != ~0)
1151             return 0;
1152           else
1153             return VNET_API_ERROR_NO_SUCH_ENTRY;
1154         }
1155
1156       if (identity_nat)
1157         {
1158           if (vrf_id == ~0)
1159             vrf_id = sm->inside_vrf_id;
1160
1161           /* *INDENT-OFF* */
1162           pool_foreach (local, m->locals,
1163           ({
1164             if (local->vrf_id == vrf_id)
1165               find = local - m->locals;
1166           }));
1167           /* *INDENT-ON* */
1168           if (find == ~0)
1169             return VNET_API_ERROR_NO_SUCH_ENTRY;
1170
1171           local = pool_elt_at_index (m->locals, find);
1172           fib_index = local->fib_index;
1173           pool_put (m->locals, local);
1174         }
1175       else
1176         fib_index = m->fib_index;
1177
1178       /* Free external address port */
1179       if (!(addr_only || sm->static_mapping_only || out2in_only))
1180         {
1181           for (i = 0; i < vec_len (sm->addresses); i++)
1182             {
1183               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1184                 {
1185                   a = sm->addresses + i;
1186                   switch (proto)
1187                     {
1188 #define _(N, j, n, s) \
1189                     case NAT_PROTOCOL_##N: \
1190                       --a->busy_##n##_port_refcounts[e_port]; \
1191                       if (e_port > 1024) \
1192                         { \
1193                           a->busy_##n##_ports--; \
1194                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1195                         } \
1196                       break;
1197                       foreach_nat_protocol
1198 #undef _
1199                     default:
1200                       nat_elog_info ("unknown protocol");
1201                       return VNET_API_ERROR_INVALID_VALUE_2;
1202                     }
1203                   break;
1204                 }
1205             }
1206         }
1207
1208       if (sm->num_workers > 1)
1209         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1210       else
1211         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1212
1213       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1214       if (!out2in_only)
1215         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1216
1217       /* Delete session(s) for static mapping if exist */
1218       if (!(sm->static_mapping_only) ||
1219           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1220         {
1221           if (sm->endpoint_dependent)
1222             {
1223               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1224                                                    m->local_port, m->proto,
1225                                                    fib_index, addr_only,
1226                                                    e_addr, e_port);
1227             }
1228           else
1229             {
1230               u_key.addr = m->local_addr;
1231               u_key.fib_index = fib_index;
1232               kv.key = u_key.as_u64;
1233               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1234                                                 e_addr, e_port);
1235             }
1236         }
1237
1238       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1239       if (pool_elts (m->locals))
1240         return 0;
1241
1242       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1243       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1244
1245       vec_free (m->tag);
1246       vec_free (m->workers);
1247       /* Delete static mapping from pool */
1248       pool_put (sm->static_mappings, m);
1249     }
1250
1251   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1252     return 0;
1253
1254   /* Add/delete external address to FIB */
1255   /* *INDENT-OFF* */
1256   pool_foreach (interface, sm->interfaces,
1257   ({
1258     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1259       continue;
1260
1261     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1262     break;
1263   }));
1264   pool_foreach (interface, sm->output_feature_interfaces,
1265   ({
1266     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1267       continue;
1268
1269     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1270     break;
1271   }));
1272   /* *INDENT-ON* */
1273
1274   return 0;
1275 }
1276
1277 int
1278 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1279                                  nat_protocol_t proto,
1280                                  nat44_lb_addr_port_t * locals, u8 is_add,
1281                                  twice_nat_type_t twice_nat, u8 out2in_only,
1282                                  u8 * tag, u32 affinity)
1283 {
1284   snat_main_t *sm = &snat_main;
1285   snat_static_mapping_t *m;
1286   clib_bihash_kv_8_8_t kv, value;
1287   snat_address_t *a = 0;
1288   int i;
1289   nat44_lb_addr_port_t *local;
1290   snat_main_per_thread_data_t *tsm;
1291   snat_session_t *s;
1292   uword *bitmap = 0;
1293
1294   if (!sm->endpoint_dependent)
1295     return VNET_API_ERROR_FEATURE_DISABLED;
1296
1297   init_nat_k (&kv, e_addr, e_port, 0, proto);
1298   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1299     m = 0;
1300   else
1301     m = pool_elt_at_index (sm->static_mappings, value.value);
1302
1303   if (is_add)
1304     {
1305       if (m)
1306         return VNET_API_ERROR_VALUE_EXIST;
1307
1308       if (vec_len (locals) < 2)
1309         return VNET_API_ERROR_INVALID_VALUE;
1310
1311       /* Find external address in allocated addresses and reserve port for
1312          address and port pair mapping when dynamic translations enabled */
1313       if (!(sm->static_mapping_only || out2in_only))
1314         {
1315           for (i = 0; i < vec_len (sm->addresses); i++)
1316             {
1317               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1318                 {
1319                   a = sm->addresses + i;
1320                   /* External port must be unused */
1321                   switch (proto)
1322                     {
1323 #define _(N, j, n, s) \
1324                     case NAT_PROTOCOL_##N: \
1325                       if (a->busy_##n##_port_refcounts[e_port]) \
1326                         return VNET_API_ERROR_INVALID_VALUE; \
1327                       ++a->busy_##n##_port_refcounts[e_port]; \
1328                       if (e_port > 1024) \
1329                         { \
1330                           a->busy_##n##_ports++; \
1331                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1332                         } \
1333                       break;
1334                       foreach_nat_protocol
1335 #undef _
1336                     default:
1337                       nat_elog_info ("unknown protocol");
1338                       return VNET_API_ERROR_INVALID_VALUE_2;
1339                     }
1340                   break;
1341                 }
1342             }
1343           /* External address must be allocated */
1344           if (!a)
1345             return VNET_API_ERROR_NO_SUCH_ENTRY;
1346         }
1347
1348       pool_get (sm->static_mappings, m);
1349       clib_memset (m, 0, sizeof (*m));
1350       m->tag = vec_dup (tag);
1351       m->external_addr = e_addr;
1352       m->external_port = e_port;
1353       m->proto = proto;
1354       m->twice_nat = twice_nat;
1355       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1356       if (out2in_only)
1357         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1358       m->affinity = affinity;
1359
1360       if (affinity)
1361         m->affinity_per_service_list_head_index =
1362           nat_affinity_get_per_service_list_head_index ();
1363       else
1364         m->affinity_per_service_list_head_index = ~0;
1365
1366       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1367                    m - sm->static_mappings);
1368       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1369         {
1370           nat_elog_err ("static_mapping_by_external key add failed");
1371           return VNET_API_ERROR_UNSPECIFIED;
1372         }
1373
1374       for (i = 0; i < vec_len (locals); i++)
1375         {
1376           locals[i].fib_index =
1377             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1378                                                locals[i].vrf_id,
1379                                                nat_fib_src_low);
1380           if (!out2in_only)
1381             {
1382               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1383                            locals[i].fib_index, m->proto,
1384                            m - sm->static_mappings);
1385               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1386             }
1387           locals[i].prefix = (i == 0) ? locals[i].probability :
1388             (locals[i - 1].prefix + locals[i].probability);
1389           pool_get (m->locals, local);
1390           *local = locals[i];
1391           if (sm->num_workers > 1)
1392             {
1393               ip4_header_t ip = {
1394                 .src_address = locals[i].addr,
1395               };
1396               bitmap =
1397                 clib_bitmap_set (bitmap,
1398                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1399                                  1);
1400             }
1401         }
1402
1403       /* Assign workers */
1404       if (sm->num_workers > 1)
1405         {
1406           /* *INDENT-OFF* */
1407           clib_bitmap_foreach (i, bitmap,
1408             ({
1409                vec_add1(m->workers, i);
1410             }));
1411           /* *INDENT-ON* */
1412         }
1413     }
1414   else
1415     {
1416       if (!m)
1417         return VNET_API_ERROR_NO_SUCH_ENTRY;
1418
1419       if (!is_lb_static_mapping (m))
1420         return VNET_API_ERROR_INVALID_VALUE;
1421
1422       /* Free external address port */
1423       if (!(sm->static_mapping_only || out2in_only))
1424         {
1425           for (i = 0; i < vec_len (sm->addresses); i++)
1426             {
1427               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1428                 {
1429                   a = sm->addresses + i;
1430                   switch (proto)
1431                     {
1432 #define _(N, j, n, s) \
1433                     case NAT_PROTOCOL_##N: \
1434                       --a->busy_##n##_port_refcounts[e_port]; \
1435                       if (e_port > 1024) \
1436                         { \
1437                           a->busy_##n##_ports--; \
1438                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1439                         } \
1440                       break;
1441                       foreach_nat_protocol
1442 #undef _
1443                     default:
1444                       nat_elog_info ("unknown protocol");
1445                       return VNET_API_ERROR_INVALID_VALUE_2;
1446                     }
1447                   break;
1448                 }
1449             }
1450         }
1451
1452       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1453       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1454         {
1455           nat_elog_err ("static_mapping_by_external key del failed");
1456           return VNET_API_ERROR_UNSPECIFIED;
1457         }
1458
1459       /* *INDENT-OFF* */
1460       pool_foreach (local, m->locals,
1461       ({
1462           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1463                             nat_fib_src_low);
1464           if (!out2in_only)
1465             {
1466 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1467               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1468                 {
1469                   nat_elog_err ("static_mapping_by_local key del failed");
1470                   return VNET_API_ERROR_UNSPECIFIED;
1471                 }
1472             }
1473
1474           if (sm->num_workers > 1)
1475             {
1476               ip4_header_t ip = {
1477                 .src_address = local->addr,
1478               };
1479               tsm = vec_elt_at_index (sm->per_thread_data,
1480                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1481             }
1482           else
1483             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1484
1485           /* Delete sessions */
1486           pool_foreach (s, tsm->sessions, {
1487             if (!(is_lb_session (s)))
1488               continue;
1489
1490             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1491                 s->in2out.port != local->port)
1492               continue;
1493
1494             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1495             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1496           });
1497       }));
1498       /* *INDENT-ON* */
1499       if (m->affinity)
1500         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1501       pool_free (m->locals);
1502       vec_free (m->tag);
1503       vec_free (m->workers);
1504
1505       pool_put (sm->static_mappings, m);
1506     }
1507
1508   return 0;
1509 }
1510
1511 int
1512 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1513                                        ip4_address_t l_addr, u16 l_port,
1514                                        nat_protocol_t proto, u32 vrf_id,
1515                                        u8 probability, u8 is_add)
1516 {
1517   snat_main_t *sm = &snat_main;
1518   snat_static_mapping_t *m = 0;
1519   clib_bihash_kv_8_8_t kv, value;
1520   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1521   snat_main_per_thread_data_t *tsm;
1522   snat_session_t *s;
1523   u32 *locals = 0;
1524   uword *bitmap = 0;
1525   int i;
1526
1527   if (!sm->endpoint_dependent)
1528     return VNET_API_ERROR_FEATURE_DISABLED;
1529
1530   init_nat_k (&kv, e_addr, e_port, 0, proto);
1531   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1532     m = pool_elt_at_index (sm->static_mappings, value.value);
1533
1534   if (!m)
1535     return VNET_API_ERROR_NO_SUCH_ENTRY;
1536
1537   if (!is_lb_static_mapping (m))
1538     return VNET_API_ERROR_INVALID_VALUE;
1539
1540   /* *INDENT-OFF* */
1541   pool_foreach (local, m->locals,
1542   ({
1543     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1544         (local->vrf_id == vrf_id))
1545       {
1546         match_local = local;
1547         break;
1548       }
1549   }));
1550   /* *INDENT-ON* */
1551
1552   if (is_add)
1553     {
1554       if (match_local)
1555         return VNET_API_ERROR_VALUE_EXIST;
1556
1557       pool_get (m->locals, local);
1558       clib_memset (local, 0, sizeof (*local));
1559       local->addr.as_u32 = l_addr.as_u32;
1560       local->port = l_port;
1561       local->probability = probability;
1562       local->vrf_id = vrf_id;
1563       local->fib_index =
1564         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1565                                            nat_fib_src_low);
1566
1567       if (!is_out2in_only_static_mapping (m))
1568         {
1569           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto,
1570                        m - sm->static_mappings);
1571           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1572             nat_elog_err ("static_mapping_by_local key add failed");
1573         }
1574     }
1575   else
1576     {
1577       if (!match_local)
1578         return VNET_API_ERROR_NO_SUCH_ENTRY;
1579
1580       if (pool_elts (m->locals) < 3)
1581         return VNET_API_ERROR_UNSPECIFIED;
1582
1583       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1584                         nat_fib_src_low);
1585
1586       if (!is_out2in_only_static_mapping (m))
1587         {
1588           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1589           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1590             nat_elog_err ("static_mapping_by_local key del failed");
1591         }
1592
1593       if (sm->num_workers > 1)
1594         {
1595           ip4_header_t ip = {
1596             .src_address = local->addr,
1597           };
1598           tsm = vec_elt_at_index (sm->per_thread_data,
1599                                   sm->worker_in2out_cb (&ip, m->fib_index,
1600                                                         0));
1601         }
1602       else
1603         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1604
1605       /* Delete sessions */
1606       /* *INDENT-OFF* */
1607       pool_foreach (s, tsm->sessions, {
1608         if (!(is_lb_session (s)))
1609           continue;
1610
1611         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1612             s->in2out.port != match_local->port)
1613           continue;
1614
1615         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1616         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1617       });
1618       /* *INDENT-ON* */
1619
1620       pool_put (m->locals, match_local);
1621     }
1622
1623   vec_free (m->workers);
1624
1625   /* *INDENT-OFF* */
1626   pool_foreach (local, m->locals,
1627   ({
1628     vec_add1 (locals, local - m->locals);
1629     if (sm->num_workers > 1)
1630       {
1631         ip4_header_t ip;
1632         ip.src_address.as_u32 = local->addr.as_u32,
1633         bitmap = clib_bitmap_set (bitmap,
1634                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1635                                   1);
1636       }
1637   }));
1638   /* *INDENT-ON* */
1639
1640   ASSERT (vec_len (locals) > 1);
1641
1642   local = pool_elt_at_index (m->locals, locals[0]);
1643   local->prefix = local->probability;
1644   for (i = 1; i < vec_len (locals); i++)
1645     {
1646       local = pool_elt_at_index (m->locals, locals[i]);
1647       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1648       local->prefix = local->probability + prev_local->prefix;
1649     }
1650
1651   /* Assign workers */
1652   if (sm->num_workers > 1)
1653     {
1654       /* *INDENT-OFF* */
1655       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1656       /* *INDENT-ON* */
1657     }
1658
1659   return 0;
1660 }
1661
1662 int
1663 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1664                   u8 twice_nat)
1665 {
1666   snat_address_t *a = 0;
1667   snat_session_t *ses;
1668   u32 *ses_to_be_removed = 0, *ses_index;
1669   snat_main_per_thread_data_t *tsm;
1670   snat_static_mapping_t *m;
1671   snat_interface_t *interface;
1672   int i;
1673   snat_address_t *addresses =
1674     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1675
1676   /* Find SNAT address */
1677   for (i = 0; i < vec_len (addresses); i++)
1678     {
1679       if (addresses[i].addr.as_u32 == addr.as_u32)
1680         {
1681           a = addresses + i;
1682           break;
1683         }
1684     }
1685   if (!a)
1686     return VNET_API_ERROR_NO_SUCH_ENTRY;
1687
1688   if (delete_sm)
1689     {
1690       /* *INDENT-OFF* */
1691       pool_foreach (m, sm->static_mappings,
1692       ({
1693           if (m->external_addr.as_u32 == addr.as_u32)
1694             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1695                                             m->local_port, m->external_port,
1696                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1697                                             m->proto, 0, m->twice_nat,
1698                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1699       }));
1700       /* *INDENT-ON* */
1701     }
1702   else
1703     {
1704       /* Check if address is used in some static mapping */
1705       if (is_snat_address_used_in_static_mapping (sm, addr))
1706         {
1707           nat_elog_notice ("address used in static mapping");
1708           return VNET_API_ERROR_UNSPECIFIED;
1709         }
1710     }
1711
1712   if (a->fib_index != ~0)
1713     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1714
1715   /* Delete sessions using address */
1716   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1717     {
1718       /* *INDENT-OFF* */
1719       vec_foreach (tsm, sm->per_thread_data)
1720         {
1721           pool_foreach (ses, tsm->sessions, ({
1722             if (ses->out2in.addr.as_u32 == addr.as_u32)
1723               {
1724                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1725                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1726               }
1727           }));
1728
1729           if (sm->endpoint_dependent){
1730               vec_foreach (ses_index, ses_to_be_removed)
1731                 {
1732                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1733                   nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1734                 }
1735           }else{
1736               vec_foreach (ses_index, ses_to_be_removed)
1737                 {
1738                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1739                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1740                 }
1741           }
1742
1743           vec_free (ses_to_be_removed);
1744         }
1745       /* *INDENT-ON* */
1746     }
1747
1748 #define _(N, i, n, s) \
1749   vec_free (a->busy_##n##_ports_per_thread);
1750   foreach_nat_protocol
1751 #undef _
1752     if (twice_nat)
1753     {
1754       vec_del1 (sm->twice_nat_addresses, i);
1755       return 0;
1756     }
1757   else
1758     vec_del1 (sm->addresses, i);
1759
1760   /* Delete external address from FIB */
1761   /* *INDENT-OFF* */
1762   pool_foreach (interface, sm->interfaces,
1763   ({
1764     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1765       continue;
1766
1767     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1768     break;
1769   }));
1770   pool_foreach (interface, sm->output_feature_interfaces,
1771   ({
1772     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1773       continue;
1774
1775     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1776     break;
1777   }));
1778   /* *INDENT-ON* */
1779
1780   return 0;
1781 }
1782
1783 static void
1784 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1785 {
1786 #define _(x)                                                                  \
1787   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1788                                 sw_if_index);                                 \
1789   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1790   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1791                                 sw_if_index);                                 \
1792   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1793   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1794                                 sw_if_index);                                 \
1795   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1796   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1797                                 sw_if_index);                                 \
1798   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1799   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1800                                 sw_if_index);                                 \
1801   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1802   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1803                                 sw_if_index);                                 \
1804   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1805   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1806                                 sw_if_index);                                 \
1807   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1808   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1809                                 sw_if_index);                                 \
1810   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1811   foreach_nat_counter;
1812 #undef _
1813   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1814   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1815 }
1816
1817 int
1818 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1819 {
1820   snat_main_t *sm = &snat_main;
1821   snat_interface_t *i;
1822   const char *feature_name, *del_feature_name;
1823   snat_address_t *ap;
1824   snat_static_mapping_t *m;
1825   nat_outside_fib_t *outside_fib;
1826   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1827                                                        sw_if_index);
1828
1829   if (sm->out2in_dpo && !is_inside)
1830     return VNET_API_ERROR_UNSUPPORTED;
1831
1832   /* *INDENT-OFF* */
1833   pool_foreach (i, sm->output_feature_interfaces,
1834   ({
1835     if (i->sw_if_index == sw_if_index)
1836       return VNET_API_ERROR_VALUE_EXIST;
1837   }));
1838   /* *INDENT-ON* */
1839
1840   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1841     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1842   else
1843     {
1844       if (sm->num_workers > 1)
1845         feature_name =
1846           is_inside ? "nat44-in2out-worker-handoff" :
1847           "nat44-out2in-worker-handoff";
1848       else if (sm->endpoint_dependent)
1849         {
1850           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1851         }
1852       else
1853         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1854     }
1855
1856   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
1857     sm->fq_in2out_index =
1858       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1859
1860   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1861     sm->fq_out2in_index =
1862       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1863
1864   if (!is_inside)
1865     {
1866       /* *INDENT-OFF* */
1867       vec_foreach (outside_fib, sm->outside_fibs)
1868         {
1869           if (outside_fib->fib_index == fib_index)
1870             {
1871               if (is_del)
1872                 {
1873                   outside_fib->refcount--;
1874                   if (!outside_fib->refcount)
1875                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1876                 }
1877               else
1878                 outside_fib->refcount++;
1879               goto feature_set;
1880             }
1881         }
1882       /* *INDENT-ON* */
1883       if (!is_del)
1884         {
1885           vec_add2 (sm->outside_fibs, outside_fib, 1);
1886           outside_fib->refcount = 1;
1887           outside_fib->fib_index = fib_index;
1888         }
1889     }
1890 feature_set:
1891   /* *INDENT-OFF* */
1892   pool_foreach (i, sm->interfaces,
1893   ({
1894     if (i->sw_if_index == sw_if_index)
1895       {
1896         if (is_del)
1897           {
1898             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1899               {
1900                 if (is_inside)
1901                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1902                 else
1903                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1904
1905                 if (sm->num_workers > 1)
1906                   {
1907                     del_feature_name = "nat44-handoff-classify";
1908                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1909                                                  "nat44-out2in-worker-handoff";
1910                   }
1911                 else if (sm->endpoint_dependent)
1912                   {
1913                     del_feature_name = "nat44-ed-classify";
1914                     feature_name = !is_inside ?  "nat-pre-in2out" :
1915                                                  "nat-pre-out2in";
1916                   }
1917                 else
1918                   {
1919                     del_feature_name = "nat44-classify";
1920                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1921                   }
1922
1923                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1924                 if (rv)
1925                   return rv;
1926                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1927                                              sw_if_index, 0, 0, 0);
1928                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1929                                              sw_if_index, 1, 0, 0);
1930                 if (!is_inside)
1931                   {
1932                     if (sm->endpoint_dependent)
1933                       vnet_feature_enable_disable ("ip4-local",
1934                                                    "nat44-ed-hairpinning",
1935                                                    sw_if_index, 1, 0, 0);
1936                     else
1937                       vnet_feature_enable_disable ("ip4-local",
1938                                                    "nat44-hairpinning",
1939                                                    sw_if_index, 1, 0, 0);
1940                   }
1941               }
1942             else
1943               {
1944                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1945                 if (rv)
1946                   return rv;
1947                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1948                                              sw_if_index, 0, 0, 0);
1949                 pool_put (sm->interfaces, i);
1950                 if (is_inside)
1951                   {
1952                     if (sm->endpoint_dependent)
1953                       vnet_feature_enable_disable ("ip4-local",
1954                                                    "nat44-ed-hairpinning",
1955                                                    sw_if_index, 0, 0, 0);
1956                     else
1957                       vnet_feature_enable_disable ("ip4-local",
1958                                                    "nat44-hairpinning",
1959                                                    sw_if_index, 0, 0, 0);
1960                   }
1961               }
1962           }
1963         else
1964           {
1965             if ((nat_interface_is_inside(i) && is_inside) ||
1966                 (nat_interface_is_outside(i) && !is_inside))
1967               return 0;
1968
1969             if (sm->num_workers > 1)
1970               {
1971                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1972                                                  "nat44-out2in-worker-handoff";
1973                 feature_name = "nat44-handoff-classify";
1974               }
1975             else if (sm->endpoint_dependent)
1976               {
1977                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
1978                                                  "nat-pre-out2in";
1979
1980                 feature_name = "nat44-ed-classify";
1981               }
1982             else
1983               {
1984                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1985                 feature_name = "nat44-classify";
1986               }
1987
1988             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1989             if (rv)
1990               return rv;
1991             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1992                                          sw_if_index, 0, 0, 0);
1993             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1994                                          sw_if_index, 1, 0, 0);
1995             if (!is_inside)
1996               {
1997                 if (sm->endpoint_dependent)
1998                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1999                                                sw_if_index, 0, 0, 0);
2000                 else
2001                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2002                                                sw_if_index, 0, 0, 0);
2003               }
2004             goto set_flags;
2005           }
2006
2007         goto fib;
2008       }
2009   }));
2010   /* *INDENT-ON* */
2011
2012   if (is_del)
2013     return VNET_API_ERROR_NO_SUCH_ENTRY;
2014
2015   pool_get (sm->interfaces, i);
2016   i->sw_if_index = sw_if_index;
2017   i->flags = 0;
2018   nat_validate_counters (sm, sw_if_index);
2019
2020   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2021                                0);
2022
2023   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2024   if (rv)
2025     return rv;
2026
2027   if (is_inside && !sm->out2in_dpo)
2028     {
2029       if (sm->endpoint_dependent)
2030         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2031                                      sw_if_index, 1, 0, 0);
2032       else
2033         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2034                                      sw_if_index, 1, 0, 0);
2035     }
2036
2037 set_flags:
2038   if (is_inside)
2039     {
2040       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2041       return 0;
2042     }
2043   else
2044     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2045
2046   /* Add/delete external addresses to FIB */
2047 fib:
2048   /* *INDENT-OFF* */
2049   vec_foreach (ap, sm->addresses)
2050     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2051
2052   pool_foreach (m, sm->static_mappings,
2053   ({
2054     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2055       continue;
2056
2057     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2058   }));
2059   /* *INDENT-ON* */
2060
2061   return 0;
2062 }
2063
2064 int
2065 snat_interface_add_del_output_feature (u32 sw_if_index,
2066                                        u8 is_inside, int is_del)
2067 {
2068   snat_main_t *sm = &snat_main;
2069   snat_interface_t *i;
2070   snat_address_t *ap;
2071   snat_static_mapping_t *m;
2072   nat_outside_fib_t *outside_fib;
2073   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2074                                                        sw_if_index);
2075
2076
2077   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2078     return VNET_API_ERROR_UNSUPPORTED;
2079
2080   /* *INDENT-OFF* */
2081   pool_foreach (i, sm->interfaces,
2082   ({
2083     if (i->sw_if_index == sw_if_index)
2084       return VNET_API_ERROR_VALUE_EXIST;
2085   }));
2086   /* *INDENT-ON* */
2087
2088   if (!is_inside)
2089     {
2090       /* *INDENT-OFF* */
2091       vec_foreach (outside_fib, sm->outside_fibs)
2092         {
2093           if (outside_fib->fib_index == fib_index)
2094             {
2095               if (is_del)
2096                 {
2097                   outside_fib->refcount--;
2098                   if (!outside_fib->refcount)
2099                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2100                 }
2101               else
2102                 outside_fib->refcount++;
2103               goto feature_set;
2104             }
2105         }
2106       /* *INDENT-ON* */
2107       if (!is_del)
2108         {
2109           vec_add2 (sm->outside_fibs, outside_fib, 1);
2110           outside_fib->refcount = 1;
2111           outside_fib->fib_index = fib_index;
2112         }
2113     }
2114
2115 feature_set:
2116   if (is_inside)
2117     {
2118       if (sm->endpoint_dependent)
2119         {
2120           int rv =
2121             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2122           if (rv)
2123             return rv;
2124           rv =
2125             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2126                                                             !is_del);
2127           if (rv)
2128             return rv;
2129           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2130                                        sw_if_index, !is_del, 0, 0);
2131           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2132                                        sw_if_index, !is_del, 0, 0);
2133         }
2134       else
2135         {
2136           int rv =
2137             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2138           if (rv)
2139             return rv;
2140           rv =
2141             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2142                                                             !is_del);
2143           if (rv)
2144             return rv;
2145           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2146                                        sw_if_index, !is_del, 0, 0);
2147           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2148                                        sw_if_index, !is_del, 0, 0);
2149         }
2150       goto fq;
2151     }
2152
2153   if (sm->num_workers > 1)
2154     {
2155       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2156       if (rv)
2157         return rv;
2158       rv =
2159         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2160       if (rv)
2161         return rv;
2162       vnet_feature_enable_disable ("ip4-unicast",
2163                                    "nat44-out2in-worker-handoff",
2164                                    sw_if_index, !is_del, 0, 0);
2165       vnet_feature_enable_disable ("ip4-output",
2166                                    "nat44-in2out-output-worker-handoff",
2167                                    sw_if_index, !is_del, 0, 0);
2168     }
2169   else
2170     {
2171       if (sm->endpoint_dependent)
2172         {
2173           int rv =
2174             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2175           if (rv)
2176             return rv;
2177           rv =
2178             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2179                                                             !is_del);
2180           if (rv)
2181             return rv;
2182           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2183                                        sw_if_index, !is_del, 0, 0);
2184           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2185                                        sw_if_index, !is_del, 0, 0);
2186         }
2187       else
2188         {
2189           int rv =
2190             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2191           if (rv)
2192             return rv;
2193           rv =
2194             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2195                                                             !is_del);
2196           if (rv)
2197             return rv;
2198           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2199                                        sw_if_index, !is_del, 0, 0);
2200           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2201                                        sw_if_index, !is_del, 0, 0);
2202         }
2203     }
2204
2205 fq:
2206   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2207     sm->fq_in2out_output_index =
2208       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2209
2210   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2211     sm->fq_out2in_index =
2212       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2213
2214   /* *INDENT-OFF* */
2215   pool_foreach (i, sm->output_feature_interfaces,
2216   ({
2217     if (i->sw_if_index == sw_if_index)
2218       {
2219         if (is_del)
2220           pool_put (sm->output_feature_interfaces, i);
2221         else
2222           return VNET_API_ERROR_VALUE_EXIST;
2223
2224         goto fib;
2225       }
2226   }));
2227   /* *INDENT-ON* */
2228
2229   if (is_del)
2230     return VNET_API_ERROR_NO_SUCH_ENTRY;
2231
2232   pool_get (sm->output_feature_interfaces, i);
2233   i->sw_if_index = sw_if_index;
2234   i->flags = 0;
2235   nat_validate_counters (sm, sw_if_index);
2236   if (is_inside)
2237     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2238   else
2239     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2240
2241   /* Add/delete external addresses to FIB */
2242 fib:
2243   if (is_inside)
2244     return 0;
2245
2246   /* *INDENT-OFF* */
2247   vec_foreach (ap, sm->addresses)
2248     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2249
2250   pool_foreach (m, sm->static_mappings,
2251   ({
2252     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2253       continue;
2254
2255     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2256   }));
2257   /* *INDENT-ON* */
2258
2259   return 0;
2260 }
2261
2262 int
2263 snat_set_workers (uword * bitmap)
2264 {
2265   snat_main_t *sm = &snat_main;
2266   int i, j = 0;
2267
2268   if (sm->num_workers < 2)
2269     return VNET_API_ERROR_FEATURE_DISABLED;
2270
2271   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2272     return VNET_API_ERROR_INVALID_WORKER;
2273
2274   vec_free (sm->workers);
2275   /* *INDENT-OFF* */
2276   clib_bitmap_foreach (i, bitmap,
2277     ({
2278       vec_add1(sm->workers, i);
2279       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2280       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2281       j++;
2282     }));
2283   /* *INDENT-ON* */
2284
2285   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2286
2287   return 0;
2288 }
2289
2290 static void
2291 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2292                          u32 old_fib_index)
2293 {
2294   snat_main_t *sm = &snat_main;
2295   nat_outside_fib_t *outside_fib;
2296   snat_interface_t *i;
2297   u8 is_add = 1;
2298   u8 match = 0;
2299
2300   if (new_fib_index == old_fib_index)
2301     return;
2302
2303   if (!vec_len (sm->outside_fibs))
2304     return;
2305
2306   /* *INDENT-OFF* */
2307   pool_foreach (i, sm->interfaces,
2308     ({
2309       if (i->sw_if_index == sw_if_index)
2310         {
2311           if (!(nat_interface_is_outside (i)))
2312             return;
2313           match = 1;
2314         }
2315     }));
2316
2317   pool_foreach (i, sm->output_feature_interfaces,
2318     ({
2319       if (i->sw_if_index == sw_if_index)
2320         {
2321           if (!(nat_interface_is_outside (i)))
2322             return;
2323           match = 1;
2324         }
2325     }));
2326   /* *INDENT-ON* */
2327
2328   if (!match)
2329     return;
2330
2331   vec_foreach (outside_fib, sm->outside_fibs)
2332   {
2333     if (outside_fib->fib_index == old_fib_index)
2334       {
2335         outside_fib->refcount--;
2336         if (!outside_fib->refcount)
2337           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2338         break;
2339       }
2340   }
2341
2342   vec_foreach (outside_fib, sm->outside_fibs)
2343   {
2344     if (outside_fib->fib_index == new_fib_index)
2345       {
2346         outside_fib->refcount++;
2347         is_add = 0;
2348         break;
2349       }
2350   }
2351
2352   if (is_add)
2353     {
2354       vec_add2 (sm->outside_fibs, outside_fib, 1);
2355       outside_fib->refcount = 1;
2356       outside_fib->fib_index = new_fib_index;
2357     }
2358 }
2359
2360 static void
2361 snat_ip4_table_bind (ip4_main_t * im,
2362                      uword opaque,
2363                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2364 {
2365   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2366 }
2367
2368 static void
2369 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2370                                        uword opaque,
2371                                        u32 sw_if_index,
2372                                        ip4_address_t * address,
2373                                        u32 address_length,
2374                                        u32 if_address_index, u32 is_delete);
2375
2376 static void
2377 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2378                                  uword opaque,
2379                                  u32 sw_if_index,
2380                                  ip4_address_t * address,
2381                                  u32 address_length,
2382                                  u32 if_address_index, u32 is_delete);
2383
2384 static int
2385 nat_alloc_addr_and_port_default (snat_address_t * addresses, u32 fib_index,
2386                                  u32 thread_index, nat_protocol_t proto,
2387                                  ip4_address_t * addr, u16 * port,
2388                                  u16 port_per_thread, u32 snat_thread_index);
2389
2390 void
2391 test_key_calc_split ()
2392 {
2393   ip4_address_t l_addr;
2394   l_addr.as_u8[0] = 1;
2395   l_addr.as_u8[1] = 1;
2396   l_addr.as_u8[2] = 1;
2397   l_addr.as_u8[3] = 1;
2398   ip4_address_t r_addr;
2399   r_addr.as_u8[0] = 2;
2400   r_addr.as_u8[1] = 2;
2401   r_addr.as_u8[2] = 2;
2402   r_addr.as_u8[3] = 2;
2403   u16 l_port = 40001;
2404   u16 r_port = 40301;
2405   u8 proto = 9;
2406   u32 fib_index = 9000001;
2407   u32 thread_index = 3000000001;
2408   u32 session_index = 3000000221;
2409   clib_bihash_kv_16_8_t kv;
2410   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2411               thread_index, session_index);
2412   ip4_address_t l_addr2;
2413   ip4_address_t r_addr2;
2414   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2415   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2416   u16 l_port2 = 0;
2417   u16 r_port2 = 0;
2418   u8 proto2 = 0;
2419   u32 fib_index2 = 0;
2420   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2421                &r_port2);
2422   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2423   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2424   ASSERT (l_port == l_port2);
2425   ASSERT (r_port == r_port2);
2426   ASSERT (proto == proto2);
2427   ASSERT (fib_index == fib_index2);
2428   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2429   ASSERT (session_index == ed_value_get_session_index (&kv));
2430
2431   fib_index = 7001;
2432   proto = 5;
2433   nat_protocol_t proto3 = ~0;
2434   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2435   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2436   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2437   ASSERT (l_port == l_port2);
2438   ASSERT (proto == proto3);
2439   ASSERT (fib_index == fib_index2);
2440 }
2441
2442 static clib_error_t *
2443 snat_init (vlib_main_t * vm)
2444 {
2445   snat_main_t *sm = &snat_main;
2446   clib_error_t *error = 0;
2447   ip4_main_t *im = &ip4_main;
2448   ip_lookup_main_t *lm = &im->lookup_main;
2449   uword *p;
2450   vlib_thread_registration_t *tr;
2451   vlib_thread_main_t *tm = vlib_get_thread_main ();
2452   uword *bitmap = 0;
2453   u32 i;
2454   ip4_add_del_interface_address_callback_t cb4;
2455   vlib_node_t *node;
2456
2457   sm->vnet_main = vnet_get_main ();
2458   sm->ip4_main = im;
2459   sm->ip4_lookup_main = lm;
2460   sm->api_main = vlibapi_get_main ();
2461   sm->first_worker_index = 0;
2462   sm->num_workers = 0;
2463   sm->workers = 0;
2464   sm->port_per_thread = 0xffff - 1024;
2465   sm->fq_in2out_index = ~0;
2466   sm->fq_in2out_output_index = ~0;
2467   sm->fq_out2in_index = ~0;
2468
2469   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2470   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2471   sm->forwarding_enabled = 0;
2472   sm->log_class = vlib_log_register_class ("nat", 0);
2473   sm->log_level = SNAT_LOG_ERROR;
2474   sm->mss_clamping = 0;
2475
2476   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2477   sm->error_node_index = node->index;
2478
2479   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2480   sm->pre_in2out_node_index = node->index;
2481   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2482   sm->pre_out2in_node_index = node->index;
2483
2484   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2485   sm->pre_in2out_node_index = node->index;
2486
2487   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2488   sm->pre_out2in_node_index = node->index;
2489
2490   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2491   sm->in2out_node_index = node->index;
2492   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2493   sm->in2out_output_node_index = node->index;
2494   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2495   sm->in2out_fast_node_index = node->index;
2496   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2497   sm->in2out_slowpath_node_index = node->index;
2498   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2499   sm->in2out_slowpath_output_node_index = node->index;
2500
2501   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2502   sm->ed_in2out_node_index = node->index;
2503   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2504   sm->ed_in2out_slowpath_node_index = node->index;
2505
2506   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2507   sm->out2in_node_index = node->index;
2508   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2509   sm->out2in_fast_node_index = node->index;
2510
2511   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2512   sm->ed_out2in_node_index = node->index;
2513   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2514   sm->ed_out2in_slowpath_node_index = node->index;
2515
2516   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2517   sm->hairpinning_node_index = node->index;
2518   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2519   sm->hairpin_dst_node_index = node->index;
2520   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2521   sm->hairpin_src_node_index = node->index;
2522   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2523   sm->ed_hairpinning_node_index = node->index;
2524   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2525   sm->ed_hairpin_dst_node_index = node->index;
2526   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2527   sm->ed_hairpin_src_node_index = node->index;
2528
2529   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2530   if (p)
2531     {
2532       tr = (vlib_thread_registration_t *) p[0];
2533       if (tr)
2534         {
2535           sm->num_workers = tr->count;
2536           sm->first_worker_index = tr->first_index;
2537         }
2538     }
2539
2540   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2541
2542   /* Use all available workers by default */
2543   if (sm->num_workers > 1)
2544     {
2545       for (i = 0; i < sm->num_workers; i++)
2546         bitmap = clib_bitmap_set (bitmap, i, 1);
2547       snat_set_workers (bitmap);
2548       clib_bitmap_free (bitmap);
2549     }
2550   else
2551     {
2552       sm->per_thread_data[0].snat_thread_index = 0;
2553     }
2554
2555   error = snat_api_init (vm, sm);
2556   if (error)
2557     return error;
2558
2559   /* Set up the interface address add/del callback */
2560   cb4.function = snat_ip4_add_del_interface_address_cb;
2561   cb4.function_opaque = 0;
2562
2563   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2564
2565   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2566   cb4.function_opaque = 0;
2567
2568   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2569
2570   nat_dpo_module_init ();
2571
2572   /* Init counters */
2573   sm->total_users.name = "total-users";
2574   sm->total_users.stat_segment_name = "/nat44/total-users";
2575   vlib_validate_simple_counter (&sm->total_users, 0);
2576   vlib_zero_simple_counter (&sm->total_users, 0);
2577   sm->total_sessions.name = "total-sessions";
2578   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2579   vlib_validate_simple_counter (&sm->total_sessions, 0);
2580   vlib_zero_simple_counter (&sm->total_sessions, 0);
2581   sm->user_limit_reached.name = "user-limit-reached";
2582   sm->user_limit_reached.stat_segment_name = "/nat44/user-limit-reached";
2583   vlib_validate_simple_counter (&sm->user_limit_reached, 0);
2584   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2585
2586 #define _(x)                                            \
2587   sm->counters.fastpath.in2out.x.name = #x;             \
2588   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2589       "/nat44/in2out/fastpath/" #x;                     \
2590   sm->counters.slowpath.in2out.x.name = #x;             \
2591   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2592       "/nat44/in2out/slowpath/" #x;                     \
2593   sm->counters.fastpath.out2in.x.name = #x;             \
2594   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2595       "/nat44/out2in/fastpath/" #x;                     \
2596   sm->counters.slowpath.out2in.x.name = #x;             \
2597   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2598       "/nat44/out2in/slowpath/" #x;                     \
2599   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2600   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2601       "/nat44/ed/in2out/fastpath/" #x;                  \
2602   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2603   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2604       "/nat44/ed/in2out/slowpath/" #x;                  \
2605   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2606   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2607       "/nat44/ed/out2in/fastpath/" #x;                  \
2608   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2609   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2610       "/nat44/ed/out2in/slowpath/" #x;
2611   foreach_nat_counter;
2612 #undef _
2613   sm->counters.hairpinning.name = "hairpinning";
2614   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2615
2616   /* Init IPFIX logging */
2617   snat_ipfix_logging_init (vm);
2618
2619   /* Init NAT64 */
2620   error = nat64_init (vm);
2621   if (error)
2622     return error;
2623
2624   ip4_table_bind_callback_t cbt4 = {
2625     .function = snat_ip4_table_bind,
2626   };
2627   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2628
2629   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2630                                         FIB_SOURCE_PRIORITY_HI,
2631                                         FIB_SOURCE_BH_SIMPLE);
2632   nat_fib_src_low = fib_source_allocate ("nat-low",
2633                                          FIB_SOURCE_PRIORITY_LOW,
2634                                          FIB_SOURCE_BH_SIMPLE);
2635
2636   test_key_calc_split ();
2637   return error;
2638 }
2639
2640 VLIB_INIT_FUNCTION (snat_init);
2641
2642 void
2643 snat_free_outside_address_and_port (snat_address_t * addresses,
2644                                     u32 thread_index,
2645                                     ip4_address_t * addr,
2646                                     u16 port, nat_protocol_t protocol)
2647 {
2648   snat_address_t *a;
2649   u32 address_index;
2650   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2651
2652   for (address_index = 0; address_index < vec_len (addresses);
2653        address_index++)
2654     {
2655       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2656         break;
2657     }
2658
2659   ASSERT (address_index < vec_len (addresses));
2660
2661   a = addresses + address_index;
2662
2663   switch (protocol)
2664     {
2665 #define _(N, i, n, s) \
2666     case NAT_PROTOCOL_##N: \
2667       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2668       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2669       a->busy_##n##_ports--; \
2670       a->busy_##n##_ports_per_thread[thread_index]--; \
2671       break;
2672       foreach_nat_protocol
2673 #undef _
2674     default:
2675       nat_elog_info ("unknown protocol");
2676       return;
2677     }
2678 }
2679
2680 static int
2681 nat_set_outside_address_and_port (snat_address_t * addresses,
2682                                   u32 thread_index, ip4_address_t addr,
2683                                   u16 port, nat_protocol_t protocol)
2684 {
2685   snat_address_t *a = 0;
2686   u32 address_index;
2687   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2688
2689   for (address_index = 0; address_index < vec_len (addresses);
2690        address_index++)
2691     {
2692       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2693         continue;
2694
2695       a = addresses + address_index;
2696       switch (protocol)
2697         {
2698 #define _(N, j, n, s) \
2699         case NAT_PROTOCOL_##N: \
2700           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2701             return VNET_API_ERROR_INSTANCE_IN_USE; \
2702           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2703           a->busy_##n##_ports_per_thread[thread_index]++; \
2704           a->busy_##n##_ports++; \
2705           return 0;
2706           foreach_nat_protocol
2707 #undef _
2708         default:
2709           nat_elog_info ("unknown protocol");
2710           return 1;
2711         }
2712     }
2713
2714   return VNET_API_ERROR_NO_SUCH_ENTRY;
2715 }
2716
2717 int
2718 snat_static_mapping_match (snat_main_t * sm,
2719                            ip4_address_t match_addr,
2720                            u16 match_port,
2721                            u32 match_fib_index,
2722                            nat_protocol_t match_protocol,
2723                            ip4_address_t * mapping_addr,
2724                            u16 * mapping_port,
2725                            u32 * mapping_fib_index,
2726                            u8 by_external,
2727                            u8 * is_addr_only,
2728                            twice_nat_type_t * twice_nat,
2729                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2730                            u8 * is_identity_nat)
2731 {
2732   clib_bihash_kv_8_8_t kv, value;
2733   snat_static_mapping_t *m;
2734   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2735   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2736   u8 backend_index;
2737   nat44_lb_addr_port_t *local;
2738
2739   if (by_external)
2740     {
2741       mapping_hash = &sm->static_mapping_by_external;
2742       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2743       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2744         {
2745           /* Try address only mapping */
2746           init_nat_k (&kv, match_addr, 0, 0, 0);
2747           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2748             return 1;
2749         }
2750
2751     }
2752   else
2753     {
2754       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2755                   match_protocol);
2756       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2757         {
2758           /* Try address only mapping */
2759           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2760           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2761             return 1;
2762         }
2763
2764     }
2765
2766   m = pool_elt_at_index (sm->static_mappings, value.value);
2767
2768   if (by_external)
2769     {
2770       if (is_lb_static_mapping (m))
2771         {
2772           if (PREDICT_FALSE (lb != 0))
2773             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2774           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2775                                                           match_addr,
2776                                                           match_protocol,
2777                                                           match_port,
2778                                                           &backend_index))
2779             {
2780               local = pool_elt_at_index (m->locals, backend_index);
2781               *mapping_addr = local->addr;
2782               *mapping_port = local->port;
2783               *mapping_fib_index = local->fib_index;
2784               goto end;
2785             }
2786           // pick locals matching this worker
2787           if (PREDICT_FALSE (sm->num_workers > 1))
2788             {
2789               u32 thread_index = vlib_get_thread_index ();
2790               /* *INDENT-OFF* */
2791               pool_foreach_index (i, m->locals,
2792               ({
2793                 local = pool_elt_at_index (m->locals, i);
2794
2795                 ip4_header_t ip = {
2796                   .src_address = local->addr,
2797                 };
2798
2799                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2800                     thread_index)
2801                   {
2802                     vec_add1 (tmp, i);
2803                   }
2804               }));
2805               /* *INDENT-ON* */
2806               ASSERT (vec_len (tmp) != 0);
2807             }
2808           else
2809             {
2810               /* *INDENT-OFF* */
2811               pool_foreach_index (i, m->locals,
2812               ({
2813                 vec_add1 (tmp, i);
2814               }));
2815               /* *INDENT-ON* */
2816             }
2817           hi = vec_len (tmp) - 1;
2818           local = pool_elt_at_index (m->locals, tmp[hi]);
2819           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2820           while (lo < hi)
2821             {
2822               mid = ((hi - lo) >> 1) + lo;
2823               local = pool_elt_at_index (m->locals, tmp[mid]);
2824               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2825             }
2826           local = pool_elt_at_index (m->locals, tmp[lo]);
2827           if (!(local->prefix >= rand))
2828             return 1;
2829           *mapping_addr = local->addr;
2830           *mapping_port = local->port;
2831           *mapping_fib_index = local->fib_index;
2832           if (m->affinity)
2833             {
2834               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2835                                                 match_protocol, match_port,
2836                                                 tmp[lo], m->affinity,
2837                                                 m->affinity_per_service_list_head_index))
2838                 nat_elog_info ("create affinity record failed");
2839             }
2840           vec_free (tmp);
2841         }
2842       else
2843         {
2844           if (PREDICT_FALSE (lb != 0))
2845             *lb = NO_LB_NAT;
2846           *mapping_fib_index = m->fib_index;
2847           *mapping_addr = m->local_addr;
2848           /* Address only mapping doesn't change port */
2849           *mapping_port = is_addr_only_static_mapping (m) ? match_port
2850             : m->local_port;
2851         }
2852     }
2853   else
2854     {
2855       *mapping_addr = m->external_addr;
2856       /* Address only mapping doesn't change port */
2857       *mapping_port = is_addr_only_static_mapping (m) ? match_port
2858         : m->external_port;
2859       *mapping_fib_index = sm->outside_fib_index;
2860     }
2861
2862 end:
2863   if (PREDICT_FALSE (is_addr_only != 0))
2864     *is_addr_only = is_addr_only_static_mapping (m);
2865
2866   if (PREDICT_FALSE (twice_nat != 0))
2867     *twice_nat = m->twice_nat;
2868
2869   if (PREDICT_FALSE (is_identity_nat != 0))
2870     *is_identity_nat = is_identity_static_mapping (m);
2871
2872   return 0;
2873 }
2874
2875 int
2876 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2877                                      u32 fib_index,
2878                                      u32 thread_index,
2879                                      nat_protocol_t proto,
2880                                      ip4_address_t * addr,
2881                                      u16 * port,
2882                                      u16 port_per_thread,
2883                                      u32 snat_thread_index)
2884 {
2885   snat_main_t *sm = &snat_main;
2886
2887   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, proto,
2888                                   addr, port, port_per_thread,
2889                                   snat_thread_index);
2890 }
2891
2892 static int
2893 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2894                                  u32 fib_index,
2895                                  u32 thread_index,
2896                                  nat_protocol_t proto,
2897                                  ip4_address_t * addr,
2898                                  u16 * port,
2899                                  u16 port_per_thread, u32 snat_thread_index)
2900 {
2901   int i;
2902   snat_address_t *a, *ga = 0;
2903   u32 portnum;
2904
2905   for (i = 0; i < vec_len (addresses); i++)
2906     {
2907       a = addresses + i;
2908       switch (proto)
2909         {
2910 #define _(N, j, n, s) \
2911         case NAT_PROTOCOL_##N: \
2912           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2913             { \
2914               if (a->fib_index == fib_index) \
2915                 { \
2916                   while (1) \
2917                     { \
2918                       portnum = (port_per_thread * \
2919                         snat_thread_index) + \
2920                         snat_random_port(0, port_per_thread - 1) + 1024; \
2921                       if (a->busy_##n##_port_refcounts[portnum]) \
2922                         continue; \
2923                       --a->busy_##n##_port_refcounts[portnum]; \
2924                       a->busy_##n##_ports_per_thread[thread_index]++; \
2925                       a->busy_##n##_ports++; \
2926                       *addr = a->addr; \
2927                       *port = clib_host_to_net_u16(portnum); \
2928                       return 0; \
2929                     } \
2930                 } \
2931               else if (a->fib_index == ~0) \
2932                 { \
2933                   ga = a; \
2934                 } \
2935             } \
2936           break;
2937           foreach_nat_protocol
2938 #undef _
2939         default:
2940           nat_elog_info ("unknown protocol");
2941           return 1;
2942         }
2943
2944     }
2945
2946   if (ga)
2947     {
2948       a = ga;
2949       switch (proto)
2950         {
2951 #define _(N, j, n, s) \
2952         case NAT_PROTOCOL_##N: \
2953           while (1) \
2954             { \
2955               portnum = (port_per_thread * \
2956                 snat_thread_index) + \
2957                 snat_random_port(0, port_per_thread - 1) + 1024; \
2958               if (a->busy_##n##_port_refcounts[portnum]) \
2959                 continue; \
2960               ++a->busy_##n##_port_refcounts[portnum]; \
2961               a->busy_##n##_ports_per_thread[thread_index]++; \
2962               a->busy_##n##_ports++; \
2963               *addr = a->addr; \
2964               *port = clib_host_to_net_u16(portnum); \
2965               return 0; \
2966             }
2967           break;
2968           foreach_nat_protocol
2969 #undef _
2970         default:
2971           nat_elog_info ("unknown protocol");
2972           return 1;
2973         }
2974     }
2975
2976   /* Totally out of translations to use... */
2977   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2978   return 1;
2979 }
2980
2981 static int
2982 nat_alloc_addr_and_port_mape (snat_address_t * addresses, u32 fib_index,
2983                               u32 thread_index, nat_protocol_t proto,
2984                               ip4_address_t * addr, u16 * port,
2985                               u16 port_per_thread, u32 snat_thread_index)
2986 {
2987   snat_main_t *sm = &snat_main;
2988   snat_address_t *a = addresses;
2989   u16 m, ports, portnum, A, j;
2990   m = 16 - (sm->psid_offset + sm->psid_length);
2991   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2992
2993   if (!vec_len (addresses))
2994     goto exhausted;
2995
2996   switch (proto)
2997     {
2998 #define _(N, i, n, s) \
2999     case NAT_PROTOCOL_##N: \
3000       if (a->busy_##n##_ports < ports) \
3001         { \
3002           while (1) \
3003             { \
3004               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3005               j = snat_random_port(0, pow2_mask(m)); \
3006               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3007               if (a->busy_##n##_port_refcounts[portnum]) \
3008                 continue; \
3009               ++a->busy_##n##_port_refcounts[portnum]; \
3010               a->busy_##n##_ports++; \
3011               *addr = a->addr; \
3012               *port = clib_host_to_net_u16 (portnum); \
3013               return 0; \
3014             } \
3015         } \
3016       break;
3017       foreach_nat_protocol
3018 #undef _
3019     default:
3020       nat_elog_info ("unknown protocol");
3021       return 1;
3022     }
3023
3024 exhausted:
3025   /* Totally out of translations to use... */
3026   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3027   return 1;
3028 }
3029
3030 static int
3031 nat_alloc_addr_and_port_range (snat_address_t * addresses, u32 fib_index,
3032                                u32 thread_index, nat_protocol_t proto,
3033                                ip4_address_t * addr, u16 * port,
3034                                u16 port_per_thread, u32 snat_thread_index)
3035 {
3036   snat_main_t *sm = &snat_main;
3037   snat_address_t *a = addresses;
3038   u16 portnum, ports;
3039
3040   ports = sm->end_port - sm->start_port + 1;
3041
3042   if (!vec_len (addresses))
3043     goto exhausted;
3044
3045   switch (proto)
3046     {
3047 #define _(N, i, n, s) \
3048     case NAT_PROTOCOL_##N: \
3049       if (a->busy_##n##_ports < ports) \
3050         { \
3051           while (1) \
3052             { \
3053               portnum = snat_random_port(sm->start_port, sm->end_port); \
3054               if (a->busy_##n##_port_refcounts[portnum]) \
3055                 continue; \
3056               ++a->busy_##n##_port_refcounts[portnum]; \
3057               a->busy_##n##_ports++; \
3058               *addr = a->addr; \
3059               *port = clib_host_to_net_u16 (portnum); \
3060               return 0; \
3061             } \
3062         } \
3063       break;
3064       foreach_nat_protocol
3065 #undef _
3066     default:
3067       nat_elog_info ("unknown protocol");
3068       return 1;
3069     }
3070
3071 exhausted:
3072   /* Totally out of translations to use... */
3073   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
3074   return 1;
3075 }
3076
3077 void
3078 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3079 {
3080   dpo_id_t dpo_v4 = DPO_INVALID;
3081   fib_prefix_t pfx = {
3082     .fp_proto = FIB_PROTOCOL_IP4,
3083     .fp_len = 32,
3084     .fp_addr.ip4.as_u32 = addr.as_u32,
3085   };
3086
3087   if (is_add)
3088     {
3089       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3090       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3091                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3092       dpo_reset (&dpo_v4);
3093     }
3094   else
3095     {
3096       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3097     }
3098 }
3099
3100 u8 *
3101 format_session_kvp (u8 * s, va_list * args)
3102 {
3103   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3104
3105   s = format (s, "%U session-index %llu", format_snat_key, v->key, v->value);
3106
3107   return s;
3108 }
3109
3110 u8 *
3111 format_static_mapping_kvp (u8 * s, va_list * args)
3112 {
3113   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3114
3115   s = format (s, "%U static-mapping-index %llu",
3116               format_snat_key, v->key, v->value);
3117
3118   return s;
3119 }
3120
3121 u8 *
3122 format_user_kvp (u8 * s, va_list * args)
3123 {
3124   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3125   snat_user_key_t k;
3126
3127   k.as_u64 = v->key;
3128
3129   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3130               k.fib_index, v->value);
3131
3132   return s;
3133 }
3134
3135 u8 *
3136 format_ed_session_kvp (u8 * s, va_list * args)
3137 {
3138   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3139
3140   u8 proto;
3141   u16 r_port, l_port;
3142   ip4_address_t l_addr, r_addr;
3143   u32 fib_index;
3144
3145   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
3146   s =
3147     format (s,
3148             "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
3149             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
3150             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
3151             format_ip_protocol, proto, fib_index,
3152             ed_value_get_session_index (v), ed_value_get_thread_index (v));
3153
3154   return s;
3155 }
3156
3157 static u32
3158 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3159                            u8 is_output)
3160 {
3161   snat_main_t *sm = &snat_main;
3162   u32 next_worker_index = 0;
3163   u32 hash;
3164
3165   next_worker_index = sm->first_worker_index;
3166   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3167     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3168
3169   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3170     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3171   else
3172     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3173
3174   return next_worker_index;
3175 }
3176
3177 static u32
3178 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3179                            u32 rx_fib_index0, u8 is_output)
3180 {
3181   snat_main_t *sm = &snat_main;
3182   udp_header_t *udp;
3183   u16 port;
3184   clib_bihash_kv_8_8_t kv, value;
3185   snat_static_mapping_t *m;
3186   u32 proto;
3187   u32 next_worker_index = 0;
3188
3189   /* first try static mappings without port */
3190   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3191     {
3192       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
3193       if (!clib_bihash_search_8_8
3194           (&sm->static_mapping_by_external, &kv, &value))
3195         {
3196           m = pool_elt_at_index (sm->static_mappings, value.value);
3197           return m->workers[0];
3198         }
3199     }
3200
3201   proto = ip_proto_to_nat_proto (ip0->protocol);
3202   udp = ip4_next_header (ip0);
3203   port = udp->dst_port;
3204
3205   /* unknown protocol */
3206   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3207     {
3208       /* use current thread */
3209       return vlib_get_thread_index ();
3210     }
3211
3212   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3213     {
3214       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3215       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3216       if (!icmp_type_is_error_message
3217           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3218         port = vnet_buffer (b)->ip.reass.l4_src_port;
3219       else
3220         {
3221           /* if error message, then it's not fragmented and we can access it */
3222           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3223           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3224           void *l4_header = ip4_next_header (inner_ip);
3225           switch (proto)
3226             {
3227             case NAT_PROTOCOL_ICMP:
3228               icmp = (icmp46_header_t *) l4_header;
3229               echo = (icmp_echo_header_t *) (icmp + 1);
3230               port = echo->identifier;
3231               break;
3232             case NAT_PROTOCOL_UDP:
3233             case NAT_PROTOCOL_TCP:
3234               port = ((tcp_udp_header_t *) l4_header)->src_port;
3235               break;
3236             default:
3237               return vlib_get_thread_index ();
3238             }
3239         }
3240     }
3241
3242   /* try static mappings with port */
3243   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3244     {
3245       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
3246       if (!clib_bihash_search_8_8
3247           (&sm->static_mapping_by_external, &kv, &value))
3248         {
3249           m = pool_elt_at_index (sm->static_mappings, value.value);
3250           return m->workers[0];
3251         }
3252     }
3253
3254   /* worker by outside port */
3255   next_worker_index = sm->first_worker_index;
3256   next_worker_index +=
3257     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3258   return next_worker_index;
3259 }
3260
3261 static u32
3262 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3263                                u8 is_output)
3264 {
3265   snat_main_t *sm = &snat_main;
3266   u32 next_worker_index = sm->first_worker_index;
3267   u32 hash;
3268
3269   clib_bihash_kv_16_8_t kv16, value16;
3270   snat_main_per_thread_data_t *tsm;
3271   udp_header_t *udp;
3272
3273   if (PREDICT_FALSE (is_output))
3274     {
3275       u32 fib_index = sm->outside_fib_index;
3276       nat_outside_fib_t *outside_fib;
3277       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3278       fib_prefix_t pfx = {
3279         .fp_proto = FIB_PROTOCOL_IP4,
3280         .fp_len = 32,
3281         .fp_addr = {
3282                     .ip4.as_u32 = ip->dst_address.as_u32,
3283                     }
3284         ,
3285       };
3286
3287       udp = ip4_next_header (ip);
3288
3289       switch (vec_len (sm->outside_fibs))
3290         {
3291         case 0:
3292           fib_index = sm->outside_fib_index;
3293           break;
3294         case 1:
3295           fib_index = sm->outside_fibs[0].fib_index;
3296           break;
3297         default:
3298             /* *INDENT-OFF* */
3299             vec_foreach (outside_fib, sm->outside_fibs)
3300               {
3301                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3302                 if (FIB_NODE_INDEX_INVALID != fei)
3303                   {
3304                     if (fib_entry_get_resolving_interface (fei) != ~0)
3305                       {
3306                         fib_index = outside_fib->fib_index;
3307                         break;
3308                       }
3309                   }
3310               }
3311             /* *INDENT-ON* */
3312           break;
3313         }
3314
3315       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3316                  udp->dst_port, fib_index, ip->protocol);
3317
3318       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3319                                                   &kv16, &value16)))
3320         {
3321           tsm =
3322             vec_elt_at_index (sm->per_thread_data,
3323                               ed_value_get_thread_index (&value16));
3324           next_worker_index += tsm->thread_index;
3325
3326           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3327                                   next_worker_index, fib_index,
3328                                   clib_net_to_host_u32 (ip->
3329                                                         src_address.as_u32),
3330                                   clib_net_to_host_u32 (ip->
3331                                                         dst_address.as_u32));
3332
3333           return next_worker_index;
3334         }
3335     }
3336
3337   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3338     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3339
3340   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3341     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3342   else
3343     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3344
3345   if (PREDICT_TRUE (!is_output))
3346     {
3347       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3348                               next_worker_index, rx_fib_index,
3349                               clib_net_to_host_u32 (ip->src_address.as_u32),
3350                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3351     }
3352   else
3353     {
3354       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3355                               next_worker_index, rx_fib_index,
3356                               clib_net_to_host_u32 (ip->src_address.as_u32),
3357                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3358     }
3359
3360   return next_worker_index;
3361 }
3362
3363 static u32
3364 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3365                                u32 rx_fib_index, u8 is_output)
3366 {
3367   snat_main_t *sm = &snat_main;
3368   clib_bihash_kv_8_8_t kv, value;
3369   clib_bihash_kv_16_8_t kv16, value16;
3370   snat_main_per_thread_data_t *tsm;
3371
3372   u32 proto, next_worker_index = 0;
3373   udp_header_t *udp;
3374   u16 port;
3375   snat_static_mapping_t *m;
3376   u32 hash;
3377
3378   proto = ip_proto_to_nat_proto (ip->protocol);
3379
3380   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3381     {
3382       udp = ip4_next_header (ip);
3383
3384       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3385                  udp->src_port, rx_fib_index, ip->protocol);
3386
3387       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3388                                                   &kv16, &value16)))
3389         {
3390           tsm =
3391             vec_elt_at_index (sm->per_thread_data,
3392                               ed_value_get_thread_index (&value16));
3393           vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
3394             ed_value_get_session_index (&value16);
3395           next_worker_index = sm->first_worker_index + tsm->thread_index;
3396           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3397                                   next_worker_index, rx_fib_index,
3398                                   clib_net_to_host_u32 (ip->
3399                                                         src_address.as_u32),
3400                                   clib_net_to_host_u32 (ip->
3401                                                         dst_address.as_u32));
3402           return next_worker_index;
3403         }
3404     }
3405   else if (proto == NAT_PROTOCOL_ICMP)
3406     {
3407       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
3408         {
3409           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3410                                                       &kv16, &value16)))
3411             {
3412               tsm =
3413                 vec_elt_at_index (sm->per_thread_data,
3414                                   ed_value_get_thread_index (&value16));
3415               next_worker_index = sm->first_worker_index + tsm->thread_index;
3416               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3417                                       next_worker_index, rx_fib_index,
3418                                       clib_net_to_host_u32 (ip->
3419                                                             src_address.as_u32),
3420                                       clib_net_to_host_u32 (ip->
3421                                                             dst_address.as_u32));
3422               return next_worker_index;
3423             }
3424         }
3425     }
3426
3427   /* first try static mappings without port */
3428   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3429     {
3430       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3431       if (!clib_bihash_search_8_8
3432           (&sm->static_mapping_by_external, &kv, &value))
3433         {
3434           m = pool_elt_at_index (sm->static_mappings, value.value);
3435           next_worker_index = m->workers[0];
3436           goto done;
3437         }
3438     }
3439
3440   /* unknown protocol */
3441   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3442     {
3443       /* use current thread */
3444       next_worker_index = vlib_get_thread_index ();
3445       goto done;
3446     }
3447
3448   udp = ip4_next_header (ip);
3449   port = udp->dst_port;
3450
3451   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3452     {
3453       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3454       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3455       if (!icmp_type_is_error_message
3456           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3457         port = vnet_buffer (b)->ip.reass.l4_src_port;
3458       else
3459         {
3460           /* if error message, then it's not fragmented and we can access it */
3461           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3462           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3463           void *l4_header = ip4_next_header (inner_ip);
3464           switch (proto)
3465             {
3466             case NAT_PROTOCOL_ICMP:
3467               icmp = (icmp46_header_t *) l4_header;
3468               echo = (icmp_echo_header_t *) (icmp + 1);
3469               port = echo->identifier;
3470               break;
3471             case NAT_PROTOCOL_UDP:
3472             case NAT_PROTOCOL_TCP:
3473               port = ((tcp_udp_header_t *) l4_header)->src_port;
3474               break;
3475             default:
3476               next_worker_index = vlib_get_thread_index ();
3477               goto done;
3478             }
3479         }
3480     }
3481
3482   /* try static mappings with port */
3483   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3484     {
3485       init_nat_k (&kv, ip->dst_address, proto, 0, port);
3486       if (!clib_bihash_search_8_8
3487           (&sm->static_mapping_by_external, &kv, &value))
3488         {
3489           m = pool_elt_at_index (sm->static_mappings, value.value);
3490           if (!is_lb_static_mapping (m))
3491             {
3492               next_worker_index = m->workers[0];
3493               goto done;
3494             }
3495
3496           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3497             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3498
3499           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3500             next_worker_index =
3501               m->workers[hash & (_vec_len (m->workers) - 1)];
3502           else
3503             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3504           goto done;
3505         }
3506     }
3507
3508   /* worker by outside port */
3509   next_worker_index = sm->first_worker_index;
3510   next_worker_index +=
3511     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3512
3513 done:
3514   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3515                           clib_net_to_host_u32 (ip->src_address.as_u32),
3516                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3517   return next_worker_index;
3518 }
3519
3520 void
3521 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3522                 ip4_address_t * out_addr, u16 out_port,
3523                 ip4_address_t * eh_addr, u16 eh_port,
3524                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3525                 u32 fib_index, u16 flags, u32 thread_index)
3526 {
3527   snat_main_t *sm = &snat_main;
3528   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3529   snat_user_t *u;
3530   snat_session_t *s;
3531   clib_bihash_kv_8_8_t kv;
3532   vlib_main_t *vm = vlib_get_main ();
3533   f64 now = vlib_time_now (vm);
3534   nat_outside_fib_t *outside_fib;
3535   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3536   fib_prefix_t pfx = {
3537     .fp_proto = FIB_PROTOCOL_IP4,
3538     .fp_len = 32,
3539     .fp_addr = {
3540                 .ip4.as_u32 = eh_addr->as_u32,
3541                 },
3542   };
3543
3544   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3545     {
3546       if (nat_set_outside_address_and_port
3547           (sm->addresses, thread_index, *out_addr, out_port, proto))
3548         return;
3549     }
3550
3551   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3552   if (!u)
3553     return;
3554
3555   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3556   if (!s)
3557     return;
3558
3559   if (sm->endpoint_dependent)
3560     {
3561       nat_ed_lru_insert (tsm, s, now, nat_proto_to_ip_proto (proto));
3562     }
3563
3564   s->out2in.addr.as_u32 = out_addr->as_u32;
3565   s->out2in.port = out_port;
3566   s->nat_proto = proto;
3567   s->last_heard = now;
3568   s->flags = flags;
3569   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3570   s->ext_host_port = eh_port;
3571   user_session_increment (sm, u, snat_is_session_static (s));
3572   switch (vec_len (sm->outside_fibs))
3573     {
3574     case 0:
3575       s->out2in.fib_index = sm->outside_fib_index;
3576       break;
3577     case 1:
3578       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3579       break;
3580     default:
3581       /* *INDENT-OFF* */
3582       vec_foreach (outside_fib, sm->outside_fibs)
3583         {
3584           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3585           if (FIB_NODE_INDEX_INVALID != fei)
3586             {
3587               if (fib_entry_get_resolving_interface (fei) != ~0)
3588                 {
3589                   s->out2in.fib_index = outside_fib->fib_index;
3590                   break;
3591                 }
3592             }
3593         }
3594       /* *INDENT-ON* */
3595       break;
3596     }
3597   init_nat_o2i_kv (&kv, s, s - tsm->sessions);
3598   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3599     nat_elog_warn ("out2in key add failed");
3600
3601   s->in2out.addr.as_u32 = in_addr->as_u32;
3602   s->in2out.port = in_port;
3603   s->in2out.fib_index = fib_index;
3604   init_nat_i2o_kv (&kv, s, s - tsm->sessions);
3605   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3606     nat_elog_warn ("in2out key add failed");
3607 }
3608
3609 void
3610 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3611                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3612                 u32 ti)
3613 {
3614   snat_main_t *sm = &snat_main;
3615   clib_bihash_kv_8_8_t kv, value;
3616   u32 thread_index;
3617   snat_session_t *s;
3618   snat_main_per_thread_data_t *tsm;
3619
3620   if (sm->num_workers > 1)
3621     thread_index =
3622       sm->first_worker_index +
3623       (sm->workers[(clib_net_to_host_u16 (out_port) -
3624                     1024) / sm->port_per_thread]);
3625   else
3626     thread_index = sm->num_workers;
3627   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3628
3629   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3630   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3631     return;
3632
3633   s = pool_elt_at_index (tsm->sessions, value.value);
3634   nat_free_session_data (sm, s, thread_index, 1);
3635   nat44_delete_session (sm, s, thread_index);
3636 }
3637
3638 void
3639 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3640                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3641                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3642 {
3643   snat_main_t *sm = &snat_main;
3644   clib_bihash_kv_8_8_t kv, value;
3645   snat_session_t *s;
3646   snat_main_per_thread_data_t *tsm;
3647
3648   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3649
3650   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3651   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3652     return;
3653
3654   s = pool_elt_at_index (tsm->sessions, value.value);
3655   s->total_pkts = total_pkts;
3656   s->total_bytes = total_bytes;
3657 }
3658
3659 void
3660 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3661                    ip4_address_t * out_addr, u16 out_port,
3662                    ip4_address_t * eh_addr, u16 eh_port,
3663                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3664                    u32 fib_index, u16 flags, u32 thread_index)
3665 {
3666   snat_main_t *sm = &snat_main;
3667   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3668   snat_session_t *s;
3669   clib_bihash_kv_16_8_t kv;
3670   vlib_main_t *vm = vlib_get_main ();
3671   f64 now = vlib_time_now (vm);
3672   nat_outside_fib_t *outside_fib;
3673   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3674   fib_prefix_t pfx = {
3675     .fp_proto = FIB_PROTOCOL_IP4,
3676     .fp_len = 32,
3677     .fp_addr = {
3678                 .ip4.as_u32 = eh_addr->as_u32,
3679                 },
3680   };
3681
3682
3683   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3684     {
3685       if (nat_set_outside_address_and_port
3686           (sm->addresses, thread_index, *out_addr, out_port, proto))
3687         return;
3688     }
3689
3690   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3691     {
3692       if (nat_set_outside_address_and_port
3693           (sm->addresses, thread_index, *ehn_addr, ehn_port, proto))
3694         return;
3695     }
3696
3697   s = nat_ed_session_alloc (sm, thread_index, now, proto);
3698   if (!s)
3699     return;
3700
3701   s->last_heard = now;
3702   s->flags = flags;
3703   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3704   s->ext_host_nat_port = s->ext_host_port = eh_port;
3705   if (is_twice_nat_session (s))
3706     {
3707       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3708       s->ext_host_nat_port = ehn_port;
3709     }
3710   switch (vec_len (sm->outside_fibs))
3711     {
3712     case 0:
3713       s->out2in.fib_index = sm->outside_fib_index;
3714       break;
3715     case 1:
3716       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3717       break;
3718     default:
3719       /* *INDENT-OFF* */
3720       vec_foreach (outside_fib, sm->outside_fibs)
3721         {
3722           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3723           if (FIB_NODE_INDEX_INVALID != fei)
3724             {
3725               if (fib_entry_get_resolving_interface (fei) != ~0)
3726                 {
3727                   s->out2in.fib_index = outside_fib->fib_index;
3728                   break;
3729                 }
3730             }
3731         }
3732       /* *INDENT-ON* */
3733       break;
3734     }
3735   s->nat_proto = proto;
3736   s->out2in.addr.as_u32 = out_addr->as_u32;
3737   s->out2in.port = out_port;
3738
3739   s->in2out.addr.as_u32 = in_addr->as_u32;
3740   s->in2out.port = in_port;
3741   s->in2out.fib_index = fib_index;
3742
3743   init_ed_kv (&kv, *in_addr, in_port, s->ext_host_nat_addr,
3744               s->ext_host_nat_port, fib_index, nat_proto_to_ip_proto (proto),
3745               thread_index, s - tsm->sessions);
3746   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3747     nat_elog_warn ("in2out key add failed");
3748
3749   init_ed_kv (&kv, *out_addr, out_port, *eh_addr, eh_port,
3750               s->out2in.fib_index, nat_proto_to_ip_proto (proto),
3751               thread_index, s - tsm->sessions);
3752   if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 1))
3753     nat_elog_warn ("out2in key add failed");
3754 }
3755
3756 void
3757 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3758                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3759                    u32 fib_index, u32 ti)
3760 {
3761   snat_main_t *sm = &snat_main;
3762   clib_bihash_kv_16_8_t kv, value;
3763   u32 thread_index;
3764   snat_session_t *s;
3765   snat_main_per_thread_data_t *tsm;
3766
3767   if (sm->num_workers > 1)
3768     thread_index =
3769       sm->first_worker_index +
3770       (sm->workers[(clib_net_to_host_u16 (out_port) -
3771                     1024) / sm->port_per_thread]);
3772   else
3773     thread_index = sm->num_workers;
3774   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3775
3776   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3777   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3778     return;
3779
3780   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3781   nat_free_session_data (sm, s, thread_index, 1);
3782   nat44_delete_session (sm, s, thread_index);
3783 }
3784
3785 void
3786 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3787                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3788                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3789                    u32 thread_index)
3790 {
3791   snat_main_t *sm = &snat_main;
3792   clib_bihash_kv_16_8_t kv, value;
3793   snat_session_t *s;
3794   snat_main_per_thread_data_t *tsm;
3795
3796   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3797
3798   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3799   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3800     return;
3801
3802   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3803   s->total_pkts = total_pkts;
3804   s->total_bytes = total_bytes;
3805 }
3806
3807 static u32
3808 nat_calc_bihash_buckets (u32 n_elts)
3809 {
3810   return 1 << (max_log2 (n_elts >> 1) + 1);
3811 }
3812
3813 static u32
3814 nat_calc_bihash_memory (u32 n_buckets, uword kv_size)
3815 {
3816   return n_buckets * (8 + kv_size * 4);
3817 }
3818
3819 void
3820 nat44_db_init (snat_main_per_thread_data_t * tsm)
3821 {
3822   snat_main_t *sm = &snat_main;
3823
3824   pool_alloc (tsm->sessions, sm->max_translations_per_thread);
3825   pool_alloc (tsm->lru_pool, sm->max_translations_per_thread);
3826
3827   dlist_elt_t *head;
3828
3829   pool_get (tsm->lru_pool, head);
3830   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3831   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3832
3833   pool_get (tsm->lru_pool, head);
3834   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3835   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3836
3837   pool_get (tsm->lru_pool, head);
3838   tsm->udp_lru_head_index = head - tsm->lru_pool;
3839   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3840
3841   pool_get (tsm->lru_pool, head);
3842   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3843   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3844
3845   pool_get (tsm->lru_pool, head);
3846   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3847   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3848
3849   if (sm->endpoint_dependent)
3850     {
3851       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3852                              sm->translation_buckets,
3853                              sm->translation_memory_size);
3854       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3855                                           format_ed_session_kvp);
3856     }
3857   else
3858     {
3859       clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3860                             sm->translation_buckets,
3861                             sm->translation_memory_size);
3862       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
3863       clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3864                             sm->translation_buckets,
3865                             sm->translation_memory_size);
3866       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
3867     }
3868
3869   // TODO: resolve static mappings (put only to !ED)
3870   pool_alloc (tsm->list_pool, sm->max_translations_per_thread);
3871   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets,
3872                         sm->user_memory_size);
3873   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3874 }
3875
3876 void
3877 nat44_db_free (snat_main_per_thread_data_t * tsm)
3878 {
3879   snat_main_t *sm = &snat_main;
3880
3881   pool_free (tsm->sessions);
3882   pool_free (tsm->lru_pool);
3883
3884   if (sm->endpoint_dependent)
3885     {
3886       clib_bihash_free_16_8 (&tsm->in2out_ed);
3887     }
3888   else
3889     {
3890       clib_bihash_free_8_8 (&tsm->in2out);
3891       clib_bihash_free_8_8 (&tsm->out2in);
3892     }
3893
3894   // TODO: resolve static mappings (put only to !ED)
3895   pool_free (tsm->users);
3896   pool_free (tsm->list_pool);
3897   clib_bihash_free_8_8 (&tsm->user_hash);
3898 }
3899
3900 void
3901 nat44_sessions_clear ()
3902 {
3903   snat_main_t *sm = &snat_main;
3904   snat_main_per_thread_data_t *tsm;
3905
3906   if (sm->endpoint_dependent)
3907     {
3908       clib_bihash_free_16_8 (&sm->out2in_ed);
3909       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
3910                              clib_max (1, sm->num_workers) *
3911                              sm->translation_buckets,
3912                              clib_max (1, sm->num_workers) *
3913                              sm->translation_memory_size);
3914       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
3915                                           format_ed_session_kvp);
3916     }
3917
3918   /* *INDENT-OFF* */
3919   vec_foreach (tsm, sm->per_thread_data)
3920     {
3921       u32 ti;
3922
3923       nat44_db_free (tsm);
3924       nat44_db_init (tsm);
3925
3926       ti = tsm->snat_thread_index;
3927       vlib_set_simple_counter (&sm->total_users, ti, 0, 0);
3928       vlib_set_simple_counter (&sm->total_sessions, ti, 0, 0);
3929     }
3930   /* *INDENT-ON* */
3931 }
3932
3933 static clib_error_t *
3934 snat_config (vlib_main_t * vm, unformat_input_t * input)
3935 {
3936   snat_main_t *sm = &snat_main;
3937   snat_main_per_thread_data_t *tsm;
3938
3939   u32 static_mapping_buckets = 1024;
3940   uword static_mapping_memory_size = 64 << 20;
3941
3942   u32 nat64_bib_buckets = 1024;
3943   u32 nat64_bib_memory_size = 128 << 20;
3944
3945   u32 nat64_st_buckets = 2048;
3946   uword nat64_st_memory_size = 256 << 20;
3947
3948   u32 max_users_per_thread = 0;
3949   u32 user_memory_size = 0;
3950   u32 max_translations_per_thread = 0;
3951   u32 translation_memory_size = 0;
3952
3953   u32 max_translations_per_user = ~0;
3954
3955   u32 outside_vrf_id = 0;
3956   u32 outside_ip6_vrf_id = 0;
3957   u32 inside_vrf_id = 0;
3958   u8 static_mapping_only = 0;
3959   u8 static_mapping_connection_tracking = 0;
3960
3961   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3962   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3963   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3964   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3965
3966   sm->out2in_dpo = 0;
3967   sm->endpoint_dependent = 0;
3968
3969   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3970     {
3971       if (unformat
3972           (input, "max translations per thread %d",
3973            &max_translations_per_thread))
3974         ;
3975       else if (unformat (input, "udp timeout %d", &udp_timeout))
3976         ;
3977       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3978         ;
3979       else if (unformat (input, "tcp transitory timeout %d",
3980                          &tcp_transitory_timeout));
3981       else if (unformat (input, "tcp established timeout %d",
3982                          &tcp_established_timeout));
3983       else if (unformat (input, "translation hash memory %d",
3984                          &translation_memory_size));
3985       else
3986         if (unformat
3987             (input, "max users per thread %d", &max_users_per_thread))
3988         ;
3989       else if (unformat (input, "user hash memory %d", &user_memory_size))
3990         ;
3991       else if (unformat (input, "max translations per user %d",
3992                          &max_translations_per_user))
3993         ;
3994       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3995         ;
3996       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3997         ;
3998       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3999         ;
4000       else if (unformat (input, "static mapping only"))
4001         {
4002           static_mapping_only = 1;
4003           if (unformat (input, "connection tracking"))
4004             static_mapping_connection_tracking = 1;
4005         }
4006       else if (unformat (input, "nat64 bib hash buckets %d",
4007                          &nat64_bib_buckets))
4008         ;
4009       else if (unformat (input, "nat64 bib hash memory %d",
4010                          &nat64_bib_memory_size))
4011         ;
4012       else
4013         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
4014         ;
4015       else if (unformat (input, "nat64 st hash memory %d",
4016                          &nat64_st_memory_size))
4017         ;
4018       else if (unformat (input, "out2in dpo"))
4019         sm->out2in_dpo = 1;
4020       else if (unformat (input, "endpoint-dependent"))
4021         sm->endpoint_dependent = 1;
4022       else
4023         return clib_error_return (0, "unknown input '%U'",
4024                                   format_unformat_error, input);
4025     }
4026
4027   if (static_mapping_only && (sm->endpoint_dependent))
4028     return clib_error_return (0,
4029                               "static mapping only mode available only for simple nat");
4030
4031   if (sm->out2in_dpo && (sm->endpoint_dependent))
4032     return clib_error_return (0,
4033                               "out2in dpo mode available only for simple nat");
4034   if (sm->endpoint_dependent && max_users_per_thread > 0)
4035     {
4036       return clib_error_return (0,
4037                                 "setting 'max users' in endpoint-dependent mode is not supported");
4038     }
4039
4040   if (sm->endpoint_dependent && max_translations_per_user != ~0)
4041     {
4042       return clib_error_return (0,
4043                                 "setting 'max translations per user' in endpoint-dependent mode is not supported");
4044     }
4045
4046   /* optionally configurable timeouts for testing purposes */
4047   sm->udp_timeout = udp_timeout;
4048   sm->tcp_transitory_timeout = tcp_transitory_timeout;
4049   sm->tcp_established_timeout = tcp_established_timeout;
4050   sm->icmp_timeout = icmp_timeout;
4051
4052   if (0 == max_users_per_thread)
4053     {
4054       max_users_per_thread = 1024;
4055     }
4056   sm->max_users_per_thread = max_users_per_thread;
4057   sm->user_buckets = nat_calc_bihash_buckets (sm->max_users_per_thread);
4058
4059   if (0 == max_translations_per_thread)
4060     {
4061       // default value based on legacy setting of load factor 10 * default
4062       // translation buckets 1024
4063       max_translations_per_thread = 10 * 1024;
4064     }
4065   sm->max_translations_per_thread = max_translations_per_thread;
4066   sm->translation_buckets =
4067     nat_calc_bihash_buckets (sm->max_translations_per_thread);
4068   if (0 == translation_memory_size)
4069     {
4070       translation_memory_size =
4071         nat_calc_bihash_memory (sm->translation_buckets,
4072                                 sizeof (clib_bihash_16_8_t));
4073     }
4074   sm->translation_memory_size = translation_memory_size;
4075   if (0 == user_memory_size)
4076     {
4077       user_memory_size =
4078         nat_calc_bihash_memory (sm->max_users_per_thread,
4079                                 sizeof (clib_bihash_8_8_t));
4080     }
4081   sm->user_memory_size = user_memory_size;
4082   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
4083
4084   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4085     sm->max_translations_per_thread : max_translations_per_user;
4086
4087   sm->outside_vrf_id = outside_vrf_id;
4088   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4089                                                              outside_vrf_id,
4090                                                              nat_fib_src_hi);
4091   sm->inside_vrf_id = inside_vrf_id;
4092   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4093                                                             inside_vrf_id,
4094                                                             nat_fib_src_hi);
4095   sm->static_mapping_only = static_mapping_only;
4096   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4097
4098   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
4099                   nat64_st_memory_size);
4100
4101   if (sm->endpoint_dependent)
4102     {
4103       sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4104       sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4105
4106       sm->in2out_node_index = nat44_ed_in2out_node.index;
4107       sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4108       sm->out2in_node_index = nat44_ed_out2in_node.index;
4109
4110       sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4111       sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4112       nat_affinity_init (vm);
4113       nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4114                    nat_ha_sref_ed_cb);
4115       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4116                              clib_max (1, sm->num_workers) *
4117                              sm->translation_buckets,
4118                              clib_max (1, sm->num_workers) *
4119                              sm->translation_memory_size);
4120       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4121                                           format_ed_session_kvp);
4122     }
4123   else
4124     {
4125       sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4126       sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4127
4128       sm->in2out_node_index = snat_in2out_node.index;
4129       sm->in2out_output_node_index = snat_in2out_output_node.index;
4130       sm->out2in_node_index = snat_out2in_node.index;
4131
4132       sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4133       sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4134       nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4135     }
4136   if (!static_mapping_only ||
4137       (static_mapping_only && static_mapping_connection_tracking))
4138     {
4139           /* *INDENT-OFF* */
4140           vec_foreach (tsm, sm->per_thread_data)
4141             {
4142               nat44_db_init (tsm);
4143             }
4144           /* *INDENT-ON* */
4145     }
4146   else
4147     {
4148       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4149       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4150     }
4151   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4152                         "static_mapping_by_local", static_mapping_buckets,
4153                         static_mapping_memory_size);
4154   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4155                                      format_static_mapping_kvp);
4156
4157   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4158                         "static_mapping_by_external",
4159                         static_mapping_buckets, static_mapping_memory_size);
4160   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4161                                      format_static_mapping_kvp);
4162
4163   return 0;
4164 }
4165
4166 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4167
4168 static void
4169 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4170                                  uword opaque,
4171                                  u32 sw_if_index,
4172                                  ip4_address_t * address,
4173                                  u32 address_length,
4174                                  u32 if_address_index, u32 is_delete)
4175 {
4176   snat_main_t *sm = &snat_main;
4177   snat_static_map_resolve_t *rp;
4178   snat_static_mapping_t *m;
4179   clib_bihash_kv_8_8_t kv, value;
4180   int i, rv;
4181   ip4_address_t l_addr;
4182
4183   for (i = 0; i < vec_len (sm->to_resolve); i++)
4184     {
4185       rp = sm->to_resolve + i;
4186       if (rp->addr_only == 0)
4187         continue;
4188       if (rp->sw_if_index == sw_if_index)
4189         goto match;
4190     }
4191
4192   return;
4193
4194 match:
4195   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
4196               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
4197   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4198     m = 0;
4199   else
4200     m = pool_elt_at_index (sm->static_mappings, value.value);
4201
4202   if (!is_delete)
4203     {
4204       /* Don't trip over lease renewal, static config */
4205       if (m)
4206         return;
4207     }
4208   else
4209     {
4210       if (!m)
4211         return;
4212     }
4213
4214   /* Indetity mapping? */
4215   if (rp->l_addr.as_u32 == 0)
4216     l_addr.as_u32 = address[0].as_u32;
4217   else
4218     l_addr.as_u32 = rp->l_addr.as_u32;
4219   /* Add the static mapping */
4220   rv = snat_add_static_mapping (l_addr,
4221                                 address[0],
4222                                 rp->l_port,
4223                                 rp->e_port,
4224                                 rp->vrf_id,
4225                                 rp->addr_only, ~0 /* sw_if_index */ ,
4226                                 rp->proto, !is_delete, rp->twice_nat,
4227                                 rp->out2in_only, rp->tag, rp->identity_nat);
4228   if (rv)
4229     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4230 }
4231
4232 static void
4233 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4234                                        uword opaque,
4235                                        u32 sw_if_index,
4236                                        ip4_address_t * address,
4237                                        u32 address_length,
4238                                        u32 if_address_index, u32 is_delete)
4239 {
4240   snat_main_t *sm = &snat_main;
4241   snat_static_map_resolve_t *rp;
4242   ip4_address_t l_addr;
4243   int i, j;
4244   int rv;
4245   u8 twice_nat = 0;
4246   snat_address_t *addresses = sm->addresses;
4247
4248   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4249     {
4250       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4251         goto match;
4252     }
4253
4254   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4255     {
4256       twice_nat = 1;
4257       addresses = sm->twice_nat_addresses;
4258       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4259         goto match;
4260     }
4261
4262   return;
4263
4264 match:
4265   if (!is_delete)
4266     {
4267       /* Don't trip over lease renewal, static config */
4268       for (j = 0; j < vec_len (addresses); j++)
4269         if (addresses[j].addr.as_u32 == address->as_u32)
4270           return;
4271
4272       (void) snat_add_address (sm, address, ~0, twice_nat);
4273       /* Scan static map resolution vector */
4274       for (j = 0; j < vec_len (sm->to_resolve); j++)
4275         {
4276           rp = sm->to_resolve + j;
4277           if (rp->addr_only)
4278             continue;
4279           /* On this interface? */
4280           if (rp->sw_if_index == sw_if_index)
4281             {
4282               /* Indetity mapping? */
4283               if (rp->l_addr.as_u32 == 0)
4284                 l_addr.as_u32 = address[0].as_u32;
4285               else
4286                 l_addr.as_u32 = rp->l_addr.as_u32;
4287               /* Add the static mapping */
4288               rv = snat_add_static_mapping (l_addr,
4289                                             address[0],
4290                                             rp->l_port,
4291                                             rp->e_port,
4292                                             rp->vrf_id,
4293                                             rp->addr_only,
4294                                             ~0 /* sw_if_index */ ,
4295                                             rp->proto,
4296                                             rp->is_add, rp->twice_nat,
4297                                             rp->out2in_only, rp->tag,
4298                                             rp->identity_nat);
4299               if (rv)
4300                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4301                                     "i4", rv);
4302             }
4303         }
4304       return;
4305     }
4306   else
4307     {
4308       (void) snat_del_address (sm, address[0], 1, twice_nat);
4309       return;
4310     }
4311 }
4312
4313
4314 int
4315 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4316                             u8 twice_nat)
4317 {
4318   ip4_main_t *ip4_main = sm->ip4_main;
4319   ip4_address_t *first_int_addr;
4320   snat_static_map_resolve_t *rp;
4321   u32 *indices_to_delete = 0;
4322   int i, j;
4323   u32 *auto_add_sw_if_indices =
4324     twice_nat ? sm->
4325     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4326
4327   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4328     );
4329
4330   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4331     {
4332       if (auto_add_sw_if_indices[i] == sw_if_index)
4333         {
4334           if (is_del)
4335             {
4336               /* if have address remove it */
4337               if (first_int_addr)
4338                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4339               else
4340                 {
4341                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4342                     {
4343                       rp = sm->to_resolve + j;
4344                       if (rp->sw_if_index == sw_if_index)
4345                         vec_add1 (indices_to_delete, j);
4346                     }
4347                   if (vec_len (indices_to_delete))
4348                     {
4349                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4350                         vec_del1 (sm->to_resolve, j);
4351                       vec_free (indices_to_delete);
4352                     }
4353                 }
4354               if (twice_nat)
4355                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4356               else
4357                 vec_del1 (sm->auto_add_sw_if_indices, i);
4358             }
4359           else
4360             return VNET_API_ERROR_VALUE_EXIST;
4361
4362           return 0;
4363         }
4364     }
4365
4366   if (is_del)
4367     return VNET_API_ERROR_NO_SUCH_ENTRY;
4368
4369   /* add to the auto-address list */
4370   if (twice_nat)
4371     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4372   else
4373     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4374
4375   /* If the address is already bound - or static - add it now */
4376   if (first_int_addr)
4377     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4378
4379   return 0;
4380 }
4381
4382 int
4383 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4384                    nat_protocol_t proto, u32 vrf_id, int is_in)
4385 {
4386   snat_main_per_thread_data_t *tsm;
4387   clib_bihash_kv_8_8_t kv, value;
4388   ip4_header_t ip;
4389   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4390   snat_session_t *s;
4391   clib_bihash_8_8_t *t;
4392
4393   if (sm->endpoint_dependent)
4394     return VNET_API_ERROR_UNSUPPORTED;
4395
4396   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4397   if (sm->num_workers > 1)
4398     tsm =
4399       vec_elt_at_index (sm->per_thread_data,
4400                         sm->worker_in2out_cb (&ip, fib_index, 0));
4401   else
4402     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4403
4404   init_nat_k (&kv, *addr, port, fib_index, proto);
4405   t = is_in ? &tsm->in2out : &tsm->out2in;
4406   if (!clib_bihash_search_8_8 (t, &kv, &value))
4407     {
4408       if (pool_is_free_index (tsm->sessions, value.value))
4409         return VNET_API_ERROR_UNSPECIFIED;
4410
4411       s = pool_elt_at_index (tsm->sessions, value.value);
4412       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4413       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4414       return 0;
4415     }
4416
4417   return VNET_API_ERROR_NO_SUCH_ENTRY;
4418 }
4419
4420 int
4421 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4422                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4423                       u32 vrf_id, int is_in)
4424 {
4425   ip4_header_t ip;
4426   clib_bihash_16_8_t *t;
4427   clib_bihash_kv_16_8_t kv, value;
4428   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4429   snat_session_t *s;
4430   snat_main_per_thread_data_t *tsm;
4431
4432   if (!sm->endpoint_dependent)
4433     return VNET_API_ERROR_FEATURE_DISABLED;
4434
4435   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4436   if (sm->num_workers > 1)
4437     tsm =
4438       vec_elt_at_index (sm->per_thread_data,
4439                         sm->worker_in2out_cb (&ip, fib_index, 0));
4440   else
4441     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4442
4443   t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
4444   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
4445   if (clib_bihash_search_16_8 (t, &kv, &value))
4446     {
4447       return VNET_API_ERROR_NO_SUCH_ENTRY;
4448     }
4449
4450   if (pool_is_free_index (tsm->sessions, value.value))
4451     return VNET_API_ERROR_UNSPECIFIED;
4452   s = pool_elt_at_index (tsm->sessions, value.value);
4453   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4454   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
4455   return 0;
4456 }
4457
4458 void
4459 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4460 {
4461   snat_main_t *sm = &snat_main;
4462
4463   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4464   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4465   sm->psid = psid;
4466   sm->psid_offset = psid_offset;
4467   sm->psid_length = psid_length;
4468 }
4469
4470 void
4471 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4472 {
4473   snat_main_t *sm = &snat_main;
4474
4475   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4476   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4477   sm->start_port = start_port;
4478   sm->end_port = end_port;
4479 }
4480
4481 void
4482 nat_set_alloc_addr_and_port_default (void)
4483 {
4484   snat_main_t *sm = &snat_main;
4485
4486   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4487   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4488 }
4489
4490 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4491                                  vlib_node_runtime_t * node,
4492                                  vlib_frame_t * frame)
4493 {
4494   return 0;
4495 }
4496
4497 /* *INDENT-OFF* */
4498 VLIB_REGISTER_NODE (nat_default_node) = {
4499   .name = "nat-default",
4500   .vector_size = sizeof (u32),
4501   .format_trace = 0,
4502   .type = VLIB_NODE_TYPE_INTERNAL,
4503   .n_errors = 0,
4504   .n_next_nodes = NAT_N_NEXT,
4505   .next_nodes = {
4506     [NAT_NEXT_DROP] = "error-drop",
4507     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4508     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4509     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4510     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4511     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4512     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4513     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4514     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4515   },
4516 };
4517 /* *INDENT-ON* */
4518
4519 /*
4520  * fd.io coding-style-patch-verification: ON
4521  *
4522  * Local Variables:
4523  * eval: (c-set-style "gnu")
4524  * End:
4525  */