src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 fib_source_t nat_fib_src_hi;
  42 fib_source_t nat_fib_src_low;
  43
  44 /* *INDENT-OFF* */
  45 /* Hook up input features */
  46 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat-pre-in2out",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-sv-reassembly-feature"),
  51 };
  52 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat-pre-out2in",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  56                                "ip4-dhcp-client-detect",
  57                                "ip4-sv-reassembly-feature"),
  58 };
  59 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-in2out-worker-handoff",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-out2in-worker-handoff",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-in2out",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  74 };
  75 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-out2in",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  79                                "ip4-dhcp-client-detect"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-classify",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  85 };
  86 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-det-in2out",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-det-out2in",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  95                                "ip4-dhcp-client-detect"),
  96 };
  97 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-det-classify",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-ed-in2out",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 106 };
 107 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-ed-out2in",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 111                                "ip4-dhcp-client-detect"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-ed-classify",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-handoff-classify",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 122 };
 123 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-in2out-fast",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 127 };
 128 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 129   .arc_name = "ip4-unicast",
 130   .node_name = "nat44-out2in-fast",
 131   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 132                                "ip4-dhcp-client-detect"),
 133 };
 134 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 135   .arc_name = "ip4-unicast",
 136   .node_name = "nat44-hairpin-dst",
 137   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 140   .arc_name = "ip4-unicast",
 141   .node_name = "nat44-ed-hairpin-dst",
 142   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 143 };
 144
 145 /* Hook up output features */
 146 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-in2out-output",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-in2out-output-worker-handoff",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 155 };
 156 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 157   .arc_name = "ip4-output",
 158   .node_name = "nat44-hairpin-src",
 159   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 162   .arc_name = "ip4-output",
 163   .node_name = "nat44-ed-in2out-output",
 164   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 165   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 166 };
 167 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 168   .arc_name = "ip4-output",
 169   .node_name = "nat44-ed-hairpin-src",
 170   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 171   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 172 };
 173
 174 /* Hook up ip4-local features */
 175 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 176 {
 177   .arc_name = "ip4-local",
 178   .node_name = "nat44-hairpinning",
 179   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 180 };
 181 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 182 {
 183   .arc_name = "ip4-local",
 184   .node_name = "nat44-ed-hairpinning",
 185   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 186 };
 187
 188
 189 VLIB_PLUGIN_REGISTER () = {
 190     .version = VPP_BUILD_VER,
 191     .description = "Network Address Translation (NAT)",
 192 };
 193 /* *INDENT-ON* */
 194
 195 void
 196 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 197                        u8 is_ha)
 198 {
 199   snat_session_key_t key;
 200   clib_bihash_kv_8_8_t kv;
 201   nat_ed_ses_key_t ed_key;
 202   clib_bihash_kv_16_8_t ed_kv;
 203   snat_main_per_thread_data_t *tsm =
 204     vec_elt_at_index (sm->per_thread_data, thread_index);
 205
 206   if (is_fwd_bypass_session (s))
 207     {
 208       if (snat_is_unk_proto_session (s))
 209         {
 210           ed_key.proto = s->in2out.port;
 211           ed_key.r_port = 0;
 212           ed_key.l_port = 0;
 213         }
 214       else
 215         {
 216           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 217           ed_key.l_port = s->in2out.port;
 218           ed_key.r_port = s->ext_host_port;
 219         }
 220       ed_key.l_addr = s->in2out.addr;
 221       ed_key.r_addr = s->ext_host_addr;
 222       ed_key.fib_index = 0;
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 226         nat_elog_warn ("in2out_ed key del failed");
 227       return;
 228     }
 229
 230   /* session lookup tables */
 231   if (is_ed_session (s))
 232     {
 233       if (is_affinity_sessions (s))
 234         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 235                              s->in2out.protocol, s->out2in.port);
 236       ed_key.l_addr = s->out2in.addr;
 237       ed_key.r_addr = s->ext_host_addr;
 238       ed_key.fib_index = s->out2in.fib_index;
 239       if (snat_is_unk_proto_session (s))
 240         {
 241           ed_key.proto = s->in2out.port;
 242           ed_key.r_port = 0;
 243           ed_key.l_port = 0;
 244         }
 245       else
 246         {
 247           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 248           ed_key.l_port = s->out2in.port;
 249           ed_key.r_port = s->ext_host_port;
 250         }
 251       ed_kv.key[0] = ed_key.as_u64[0];
 252       ed_kv.key[1] = ed_key.as_u64[1];
 253       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 254         nat_elog_warn ("out2in_ed key del failed");
 255       ed_key.l_addr = s->in2out.addr;
 256       ed_key.fib_index = s->in2out.fib_index;
 257       if (!snat_is_unk_proto_session (s))
 258         ed_key.l_port = s->in2out.port;
 259       if (is_twice_nat_session (s))
 260         {
 261           ed_key.r_addr = s->ext_host_nat_addr;
 262           ed_key.r_port = s->ext_host_nat_port;
 263         }
 264       ed_kv.key[0] = ed_key.as_u64[0];
 265       ed_kv.key[1] = ed_key.as_u64[1];
 266       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 267         nat_elog_warn ("in2out_ed key del failed");
 268
 269       if (!is_ha)
 270         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 271                                &s->in2out.addr, s->in2out.port,
 272                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 273                                &s->out2in.addr, s->out2in.port,
 274                                &s->ext_host_addr, s->ext_host_port,
 275                                s->in2out.protocol, is_twice_nat_session (s));
 276     }
 277   else
 278     {
 279       kv.key = s->in2out.as_u64;
 280       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 281         nat_elog_warn ("in2out key del failed");
 282       kv.key = s->out2in.as_u64;
 283       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 284         nat_elog_warn ("out2in key del failed");
 285
 286       if (!is_ha)
 287         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 288                                  &s->in2out.addr, s->in2out.port,
 289                                  &s->out2in.addr, s->out2in.port,
 290                                  s->in2out.protocol);
 291     }
 292
 293   if (snat_is_unk_proto_session (s))
 294     return;
 295
 296   if (!is_ha)
 297     {
 298       /* log NAT event */
 299       snat_ipfix_logging_nat44_ses_delete (thread_index,
 300                                            s->in2out.addr.as_u32,
 301                                            s->out2in.addr.as_u32,
 302                                            s->in2out.protocol,
 303                                            s->in2out.port,
 304                                            s->out2in.port,
 305                                            s->in2out.fib_index);
 306
 307       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 308                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 309                    thread_index);
 310     }
 311
 312   /* Twice NAT address and port for external host */
 313   if (is_twice_nat_session (s))
 314     {
 315       key.protocol = s->in2out.protocol;
 316       key.port = s->ext_host_nat_port;
 317       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 318       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 319                                           thread_index, &key);
 320     }
 321
 322   if (snat_is_session_static (s))
 323     return;
 324
 325   snat_free_outside_address_and_port (sm->addresses, thread_index,
 326                                       &s->out2in);
 327 }
 328
 329 snat_user_t *
 330 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 331                         u32 thread_index)
 332 {
 333   snat_user_t *u = 0;
 334   snat_user_key_t user_key;
 335   clib_bihash_kv_8_8_t kv, value;
 336   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 337   dlist_elt_t *per_user_list_head_elt;
 338
 339   user_key.addr.as_u32 = addr->as_u32;
 340   user_key.fib_index = fib_index;
 341   kv.key = user_key.as_u64;
 342
 343   /* Ever heard of the "user" = src ip4 address before? */
 344   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 345     {
 346       /* no, make a new one */
 347       pool_get (tsm->users, u);
 348       clib_memset (u, 0, sizeof (*u));
 349       u->addr.as_u32 = addr->as_u32;
 350       u->fib_index = fib_index;
 351
 352       pool_get (tsm->list_pool, per_user_list_head_elt);
 353
 354       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 355         tsm->list_pool;
 356
 357       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 358
 359       kv.value = u - tsm->users;
 360
 361       /* add user */
 362       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 363         nat_elog_warn ("user_hash keay add failed");
 364
 365       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 366                                pool_elts (tsm->users));
 367     }
 368   else
 369     {
 370       u = pool_elt_at_index (tsm->users, value.value);
 371     }
 372
 373   return u;
 374 }
 375
 376 snat_session_t *
 377 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 378                               u32 thread_index, f64 now)
 379 {
 380   snat_session_t *s;
 381   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 382   u32 oldest_per_user_translation_list_index, session_index;
 383   dlist_elt_t *oldest_per_user_translation_list_elt;
 384   dlist_elt_t *per_user_translation_list_elt;
 385
 386   /* Over quota? Recycle the least recently used translation */
 387   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 388     {
 389       oldest_per_user_translation_list_index =
 390         clib_dlist_remove_head (tsm->list_pool,
 391                                 u->sessions_per_user_list_head_index);
 392
 393       ASSERT (oldest_per_user_translation_list_index != ~0);
 394
 395       /* Add it back to the end of the LRU list */
 396       clib_dlist_addtail (tsm->list_pool,
 397                           u->sessions_per_user_list_head_index,
 398                           oldest_per_user_translation_list_index);
 399       /* Get the list element */
 400       oldest_per_user_translation_list_elt =
 401         pool_elt_at_index (tsm->list_pool,
 402                            oldest_per_user_translation_list_index);
 403
 404       /* Get the session index from the list element */
 405       session_index = oldest_per_user_translation_list_elt->value;
 406
 407       /* Get the session */
 408       s = pool_elt_at_index (tsm->sessions, session_index);
 409       nat_free_session_data (sm, s, thread_index, 0);
 410       if (snat_is_session_static (s))
 411         u->nstaticsessions--;
 412       else
 413         u->nsessions--;
 414       s->flags = 0;
 415       s->total_bytes = 0;
 416       s->total_pkts = 0;
 417       s->state = 0;
 418       s->ext_host_addr.as_u32 = 0;
 419       s->ext_host_port = 0;
 420       s->ext_host_nat_addr.as_u32 = 0;
 421       s->ext_host_nat_port = 0;
 422     }
 423   else
 424     {
 425       pool_get (tsm->sessions, s);
 426       clib_memset (s, 0, sizeof (*s));
 427
 428       /* Create list elts */
 429       pool_get (tsm->list_pool, per_user_translation_list_elt);
 430       clib_dlist_init (tsm->list_pool,
 431                        per_user_translation_list_elt - tsm->list_pool);
 432
 433       per_user_translation_list_elt->value = s - tsm->sessions;
 434       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 435       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 436
 437       clib_dlist_addtail (tsm->list_pool,
 438                           s->per_user_list_head_index,
 439                           per_user_translation_list_elt - tsm->list_pool);
 440
 441       s->user_index = u - tsm->users;
 442       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 443                                pool_elts (tsm->sessions));
 444     }
 445
 446   s->ha_last_refreshed = now;
 447
 448   return s;
 449 }
 450
 451 snat_session_t *
 452 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 453                       f64 now)
 454 {
 455   snat_session_t *s;
 456   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 457   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 458   u32 oldest_index;
 459   u64 sess_timeout_time;
 460
 461   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 462     goto alloc_new;
 463
 464   oldest_index =
 465     clib_dlist_remove_head (tsm->list_pool,
 466                             u->sessions_per_user_list_head_index);
 467   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 468   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 469
 470   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 471   if (now >= sess_timeout_time)
 472     {
 473       clib_dlist_addtail (tsm->list_pool,
 474                           u->sessions_per_user_list_head_index, oldest_index);
 475       nat_free_session_data (sm, s, thread_index, 0);
 476       if (snat_is_session_static (s))
 477         u->nstaticsessions--;
 478       else
 479         u->nsessions--;
 480       s->flags = 0;
 481       s->total_bytes = 0;
 482       s->total_pkts = 0;
 483       s->state = 0;
 484       s->ext_host_addr.as_u32 = 0;
 485       s->ext_host_port = 0;
 486       s->ext_host_nat_addr.as_u32 = 0;
 487       s->ext_host_nat_port = 0;
 488     }
 489   else
 490     {
 491       clib_dlist_addhead (tsm->list_pool,
 492                           u->sessions_per_user_list_head_index, oldest_index);
 493       if ((u->nsessions + u->nstaticsessions) >=
 494           sm->max_translations_per_user)
 495         {
 496           nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 497                          clib_net_to_host_u32 (u->addr.as_u32));
 498           snat_ipfix_logging_max_entries_per_user
 499             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 500           return 0;
 501         }
 502       else
 503         {
 504         alloc_new:
 505           pool_get (tsm->sessions, s);
 506           clib_memset (s, 0, sizeof (*s));
 507
 508           /* Create list elts */
 509           pool_get (tsm->list_pool, per_user_translation_list_elt);
 510           clib_dlist_init (tsm->list_pool,
 511                            per_user_translation_list_elt - tsm->list_pool);
 512
 513           per_user_translation_list_elt->value = s - tsm->sessions;
 514           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 515           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 516
 517           clib_dlist_addtail (tsm->list_pool,
 518                               s->per_user_list_head_index,
 519                               per_user_translation_list_elt - tsm->list_pool);
 520         }
 521
 522       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 523                                pool_elts (tsm->sessions));
 524     }
 525
 526   s->ha_last_refreshed = now;
 527
 528   return s;
 529 }
 530
 531 void
 532 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 533                           int is_add)
 534 {
 535   fib_prefix_t prefix = {
 536     .fp_len = p_len,
 537     .fp_proto = FIB_PROTOCOL_IP4,
 538     .fp_addr = {
 539                 .ip4.as_u32 = addr->as_u32,
 540                 },
 541   };
 542   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 543
 544   if (is_add)
 545     fib_table_entry_update_one_path (fib_index,
 546                                      &prefix,
 547                                      nat_fib_src_low,
 548                                      (FIB_ENTRY_FLAG_CONNECTED |
 549                                       FIB_ENTRY_FLAG_LOCAL |
 550                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 551                                      DPO_PROTO_IP4,
 552                                      NULL,
 553                                      sw_if_index,
 554                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 555   else
 556     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 557 }
 558
 559 int
 560 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 561                   u8 twice_nat)
 562 {
 563   snat_address_t *ap;
 564   snat_interface_t *i;
 565   vlib_thread_main_t *tm = vlib_get_thread_main ();
 566
 567   if (twice_nat && !sm->endpoint_dependent)
 568     return VNET_API_ERROR_FEATURE_DISABLED;
 569
 570   /* Check if address already exists */
 571   /* *INDENT-OFF* */
 572   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 573     {
 574       if (ap->addr.as_u32 == addr->as_u32)
 575         return VNET_API_ERROR_VALUE_EXIST;
 576     }
 577   /* *INDENT-ON* */
 578
 579   if (twice_nat)
 580     vec_add2 (sm->twice_nat_addresses, ap, 1);
 581   else
 582     vec_add2 (sm->addresses, ap, 1);
 583
 584   ap->addr = *addr;
 585   if (vrf_id != ~0)
 586     ap->fib_index =
 587       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 588                                          nat_fib_src_low);
 589   else
 590     ap->fib_index = ~0;
 591 #define _(N, i, n, s) \
 592   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 593   ap->busy_##n##_ports = 0; \
 594   ap->busy_##n##_ports_per_thread = 0;\
 595   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 596   foreach_snat_protocol
 597 #undef _
 598     if (twice_nat)
 599     return 0;
 600
 601   /* Add external address to FIB */
 602   /* *INDENT-OFF* */
 603   pool_foreach (i, sm->interfaces,
 604   ({
 605     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 606       continue;
 607
 608     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 609     break;
 610   }));
 611   pool_foreach (i, sm->output_feature_interfaces,
 612   ({
 613     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 614       continue;
 615
 616     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 617     break;
 618   }));
 619   /* *INDENT-ON* */
 620
 621   return 0;
 622 }
 623
 624 static int
 625 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 626 {
 627   snat_static_mapping_t *m;
 628   /* *INDENT-OFF* */
 629   pool_foreach (m, sm->static_mappings,
 630   ({
 631       if (is_addr_only_static_mapping (m) ||
 632           is_out2in_only_static_mapping (m) ||
 633           is_identity_static_mapping (m))
 634         continue;
 635       if (m->external_addr.as_u32 == addr.as_u32)
 636         return 1;
 637   }));
 638   /* *INDENT-ON* */
 639
 640   return 0;
 641 }
 642
 643 void
 644 increment_v4_address (ip4_address_t * a)
 645 {
 646   u32 v;
 647
 648   v = clib_net_to_host_u32 (a->as_u32) + 1;
 649   a->as_u32 = clib_host_to_net_u32 (v);
 650 }
 651
 652 static void
 653 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 654                                        ip4_address_t l_addr,
 655                                        u16 l_port,
 656                                        u32 sw_if_index,
 657                                        u16 e_port,
 658                                        u32 vrf_id,
 659                                        snat_protocol_t proto,
 660                                        int addr_only, int is_add, u8 * tag,
 661                                        int twice_nat, int out2in_only,
 662                                        int identity_nat)
 663 {
 664   snat_static_map_resolve_t *rp;
 665
 666   vec_add2 (sm->to_resolve, rp, 1);
 667   rp->l_addr.as_u32 = l_addr.as_u32;
 668   rp->l_port = l_port;
 669   rp->sw_if_index = sw_if_index;
 670   rp->e_port = e_port;
 671   rp->vrf_id = vrf_id;
 672   rp->proto = proto;
 673   rp->addr_only = addr_only;
 674   rp->is_add = is_add;
 675   rp->twice_nat = twice_nat;
 676   rp->out2in_only = out2in_only;
 677   rp->identity_nat = identity_nat;
 678   rp->tag = vec_dup (tag);
 679 }
 680
 681 static u32
 682 get_thread_idx_by_port (u16 e_port)
 683 {
 684   snat_main_t *sm = &snat_main;
 685   u32 thread_idx = sm->num_workers;
 686   if (sm->num_workers > 1)
 687     {
 688       thread_idx =
 689         sm->first_worker_index +
 690         sm->workers[(e_port - 1024) / sm->port_per_thread];
 691     }
 692   return thread_idx;
 693 }
 694
 695 int
 696 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 697                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 698                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 699                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 700                          u8 identity_nat)
 701 {
 702   snat_main_t *sm = &snat_main;
 703   snat_static_mapping_t *m;
 704   snat_session_key_t m_key;
 705   clib_bihash_kv_8_8_t kv, value;
 706   snat_address_t *a = 0;
 707   u32 fib_index = ~0;
 708   snat_interface_t *interface;
 709   int i;
 710   snat_main_per_thread_data_t *tsm;
 711   snat_user_key_t u_key;
 712   snat_user_t *u;
 713   dlist_elt_t *head, *elt;
 714   u32 elt_index, head_index;
 715   u32 ses_index;
 716   u64 user_index;
 717   snat_session_t *s;
 718   snat_static_map_resolve_t *rp, *rp_match = 0;
 719   nat44_lb_addr_port_t *local;
 720   u32 find = ~0;
 721
 722   if (!sm->endpoint_dependent)
 723     {
 724       if (twice_nat || out2in_only)
 725         return VNET_API_ERROR_FEATURE_DISABLED;
 726     }
 727
 728   /* If the external address is a specific interface address */
 729   if (sw_if_index != ~0)
 730     {
 731       ip4_address_t *first_int_addr;
 732
 733       for (i = 0; i < vec_len (sm->to_resolve); i++)
 734         {
 735           rp = sm->to_resolve + i;
 736           if (rp->sw_if_index != sw_if_index ||
 737               rp->l_addr.as_u32 != l_addr.as_u32 ||
 738               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 739             continue;
 740
 741           if (!addr_only)
 742             {
 743               if ((rp->l_port != l_port && rp->e_port != e_port)
 744                   || rp->proto != proto)
 745                 continue;
 746             }
 747
 748           rp_match = rp;
 749           break;
 750         }
 751
 752       /* Might be already set... */
 753       first_int_addr = ip4_interface_first_address
 754         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 755
 756       if (is_add)
 757         {
 758           if (rp_match)
 759             return VNET_API_ERROR_VALUE_EXIST;
 760
 761           snat_add_static_mapping_when_resolved
 762             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 763              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 764
 765           /* DHCP resolution required? */
 766           if (first_int_addr == 0)
 767             {
 768               return 0;
 769             }
 770           else
 771             {
 772               e_addr.as_u32 = first_int_addr->as_u32;
 773               /* Identity mapping? */
 774               if (l_addr.as_u32 == 0)
 775                 l_addr.as_u32 = e_addr.as_u32;
 776             }
 777         }
 778       else
 779         {
 780           if (!rp_match)
 781             return VNET_API_ERROR_NO_SUCH_ENTRY;
 782
 783           vec_del1 (sm->to_resolve, i);
 784
 785           if (first_int_addr)
 786             {
 787               e_addr.as_u32 = first_int_addr->as_u32;
 788               /* Identity mapping? */
 789               if (l_addr.as_u32 == 0)
 790                 l_addr.as_u32 = e_addr.as_u32;
 791             }
 792           else
 793             return 0;
 794         }
 795     }
 796
 797   m_key.addr = e_addr;
 798   m_key.port = addr_only ? 0 : e_port;
 799   m_key.protocol = addr_only ? 0 : proto;
 800   m_key.fib_index = 0;
 801   kv.key = m_key.as_u64;
 802   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 803     m = 0;
 804   else
 805     m = pool_elt_at_index (sm->static_mappings, value.value);
 806
 807   if (is_add)
 808     {
 809       if (m)
 810         {
 811           if (is_identity_static_mapping (m))
 812             {
 813               /* *INDENT-OFF* */
 814               pool_foreach (local, m->locals,
 815               ({
 816                 if (local->vrf_id == vrf_id)
 817                   return VNET_API_ERROR_VALUE_EXIST;
 818               }));
 819               /* *INDENT-ON* */
 820               pool_get (m->locals, local);
 821               local->vrf_id = vrf_id;
 822               local->fib_index =
 823                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 824                                                    nat_fib_src_low);
 825               m_key.addr = m->local_addr;
 826               m_key.port = m->local_port;
 827               m_key.protocol = m->proto;
 828               m_key.fib_index = local->fib_index;
 829               kv.key = m_key.as_u64;
 830               kv.value = m - sm->static_mappings;
 831               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 832               return 0;
 833             }
 834           else
 835             return VNET_API_ERROR_VALUE_EXIST;
 836         }
 837
 838       if (twice_nat && addr_only)
 839         return VNET_API_ERROR_UNSUPPORTED;
 840
 841       /* Convert VRF id to FIB index */
 842       if (vrf_id != ~0)
 843         fib_index =
 844           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 845                                              nat_fib_src_low);
 846       /* If not specified use inside VRF id from SNAT plugin startup config */
 847       else
 848         {
 849           fib_index = sm->inside_fib_index;
 850           vrf_id = sm->inside_vrf_id;
 851           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 852         }
 853
 854       if (!(out2in_only || identity_nat))
 855         {
 856           m_key.addr = l_addr;
 857           m_key.port = addr_only ? 0 : l_port;
 858           m_key.protocol = addr_only ? 0 : proto;
 859           m_key.fib_index = fib_index;
 860           kv.key = m_key.as_u64;
 861           if (!clib_bihash_search_8_8
 862               (&sm->static_mapping_by_local, &kv, &value))
 863             return VNET_API_ERROR_VALUE_EXIST;
 864         }
 865
 866       /* Find external address in allocated addresses and reserve port for
 867          address and port pair mapping when dynamic translations enabled */
 868       if (!(addr_only || sm->static_mapping_only || out2in_only))
 869         {
 870           for (i = 0; i < vec_len (sm->addresses); i++)
 871             {
 872               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 873                 {
 874                   a = sm->addresses + i;
 875                   /* External port must be unused */
 876                   switch (proto)
 877                     {
 878 #define _(N, j, n, s) \
 879                     case SNAT_PROTOCOL_##N: \
 880                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 881                         return VNET_API_ERROR_INVALID_VALUE; \
 882                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 883                       if (e_port > 1024) \
 884                         { \
 885                           a->busy_##n##_ports++; \
 886                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 887                         } \
 888                       break;
 889                       foreach_snat_protocol
 890 #undef _
 891                     default:
 892                       nat_elog_info ("unknown protocol");
 893                       return VNET_API_ERROR_INVALID_VALUE_2;
 894                     }
 895                   break;
 896                 }
 897             }
 898           /* External address must be allocated */
 899           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 900             {
 901               if (sw_if_index != ~0)
 902                 {
 903                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 904                     {
 905                       rp = sm->to_resolve + i;
 906                       if (rp->addr_only)
 907                         continue;
 908                       if (rp->sw_if_index != sw_if_index &&
 909                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 910                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 911                           rp->e_port != e_port && rp->proto != proto)
 912                         continue;
 913
 914                       vec_del1 (sm->to_resolve, i);
 915                       break;
 916                     }
 917                 }
 918               return VNET_API_ERROR_NO_SUCH_ENTRY;
 919             }
 920         }
 921
 922       pool_get (sm->static_mappings, m);
 923       clib_memset (m, 0, sizeof (*m));
 924       m->tag = vec_dup (tag);
 925       m->local_addr = l_addr;
 926       m->external_addr = e_addr;
 927       m->twice_nat = twice_nat;
 928       if (out2in_only)
 929         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 930       if (addr_only)
 931         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 932       if (identity_nat)
 933         {
 934           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 935           pool_get (m->locals, local);
 936           local->vrf_id = vrf_id;
 937           local->fib_index = fib_index;
 938         }
 939       else
 940         {
 941           m->vrf_id = vrf_id;
 942           m->fib_index = fib_index;
 943         }
 944       if (!addr_only)
 945         {
 946           m->local_port = l_port;
 947           m->external_port = e_port;
 948           m->proto = proto;
 949         }
 950
 951       if (sm->num_workers > 1)
 952         {
 953           ip4_header_t ip = {
 954             .src_address = m->local_addr,
 955           };
 956           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 957           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 958         }
 959       else
 960         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 961
 962       m_key.addr = m->local_addr;
 963       m_key.port = m->local_port;
 964       m_key.protocol = m->proto;
 965       m_key.fib_index = fib_index;
 966       kv.key = m_key.as_u64;
 967       kv.value = m - sm->static_mappings;
 968       if (!out2in_only)
 969         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 970
 971       m_key.addr = m->external_addr;
 972       m_key.port = m->external_port;
 973       m_key.fib_index = 0;
 974       kv.key = m_key.as_u64;
 975       kv.value = m - sm->static_mappings;
 976       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 977
 978       /* Delete dynamic sessions matching local address (+ local port) */
 979       if (!(sm->static_mapping_only))
 980         {
 981           u_key.addr = m->local_addr;
 982           u_key.fib_index = m->fib_index;
 983           kv.key = u_key.as_u64;
 984           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 985             {
 986               user_index = value.value;
 987               u = pool_elt_at_index (tsm->users, user_index);
 988               if (u->nsessions)
 989                 {
 990                   head_index = u->sessions_per_user_list_head_index;
 991                   head = pool_elt_at_index (tsm->list_pool, head_index);
 992                   elt_index = head->next;
 993                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 994                   ses_index = elt->value;
 995                   while (ses_index != ~0)
 996                     {
 997                       s = pool_elt_at_index (tsm->sessions, ses_index);
 998                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 999                       ses_index = elt->value;
1000
1001                       if (snat_is_session_static (s))
1002                         continue;
1003
1004                       if (!addr_only
1005                           && (clib_net_to_host_u16 (s->in2out.port) !=
1006                               m->local_port))
1007                         continue;
1008
1009                       nat_free_session_data (sm, s,
1010                                              tsm - sm->per_thread_data, 0);
1011                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1012
1013                       if (!addr_only && !sm->endpoint_dependent)
1014                         break;
1015                     }
1016                 }
1017             }
1018         }
1019     }
1020   else
1021     {
1022       if (!m)
1023         {
1024           if (sw_if_index != ~0)
1025             return 0;
1026           else
1027             return VNET_API_ERROR_NO_SUCH_ENTRY;
1028         }
1029
1030       if (identity_nat)
1031         {
1032           if (vrf_id == ~0)
1033             vrf_id = sm->inside_vrf_id;
1034
1035           /* *INDENT-OFF* */
1036           pool_foreach (local, m->locals,
1037           ({
1038             if (local->vrf_id == vrf_id)
1039               find = local - m->locals;
1040           }));
1041           /* *INDENT-ON* */
1042           if (find == ~0)
1043             return VNET_API_ERROR_NO_SUCH_ENTRY;
1044
1045           local = pool_elt_at_index (m->locals, find);
1046           fib_index = local->fib_index;
1047           pool_put (m->locals, local);
1048         }
1049       else
1050         fib_index = m->fib_index;
1051
1052       /* Free external address port */
1053       if (!(addr_only || sm->static_mapping_only || out2in_only))
1054         {
1055           for (i = 0; i < vec_len (sm->addresses); i++)
1056             {
1057               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1058                 {
1059                   a = sm->addresses + i;
1060                   switch (proto)
1061                     {
1062 #define _(N, j, n, s) \
1063                     case SNAT_PROTOCOL_##N: \
1064                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1065                       if (e_port > 1024) \
1066                         { \
1067                           a->busy_##n##_ports--; \
1068                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1069                         } \
1070                       break;
1071                       foreach_snat_protocol
1072 #undef _
1073                     default:
1074                       nat_elog_info ("unknown protocol");
1075                       return VNET_API_ERROR_INVALID_VALUE_2;
1076                     }
1077                   break;
1078                 }
1079             }
1080         }
1081
1082       if (sm->num_workers > 1)
1083         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1084       else
1085         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1086
1087       m_key.addr = m->local_addr;
1088       m_key.port = m->local_port;
1089       m_key.protocol = m->proto;
1090       m_key.fib_index = fib_index;
1091       kv.key = m_key.as_u64;
1092       if (!out2in_only)
1093         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1094
1095       /* Delete session(s) for static mapping if exist */
1096       if (!(sm->static_mapping_only) ||
1097           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1098         {
1099           u_key.addr = m->local_addr;
1100           u_key.fib_index = fib_index;
1101           kv.key = u_key.as_u64;
1102           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1103             {
1104               user_index = value.value;
1105               u = pool_elt_at_index (tsm->users, user_index);
1106               if (u->nstaticsessions)
1107                 {
1108                   head_index = u->sessions_per_user_list_head_index;
1109                   head = pool_elt_at_index (tsm->list_pool, head_index);
1110                   elt_index = head->next;
1111                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1112                   ses_index = elt->value;
1113                   while (ses_index != ~0)
1114                     {
1115                       s = pool_elt_at_index (tsm->sessions, ses_index);
1116                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1117                       ses_index = elt->value;
1118
1119                       if (!addr_only)
1120                         {
1121                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1122                               (clib_net_to_host_u16 (s->out2in.port) !=
1123                                e_port))
1124                             continue;
1125                         }
1126
1127                       if (is_lb_session (s))
1128                         continue;
1129
1130                       if (!snat_is_session_static (s))
1131                         continue;
1132
1133                       nat_free_session_data (sm, s,
1134                                              tsm - sm->per_thread_data, 0);
1135                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1136
1137                       if (!addr_only && !sm->endpoint_dependent)
1138                         break;
1139                     }
1140                 }
1141             }
1142         }
1143
1144       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1145       if (pool_elts (m->locals))
1146         return 0;
1147
1148       m_key.addr = m->external_addr;
1149       m_key.port = m->external_port;
1150       m_key.fib_index = 0;
1151       kv.key = m_key.as_u64;
1152       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1153
1154       vec_free (m->tag);
1155       vec_free (m->workers);
1156       /* Delete static mapping from pool */
1157       pool_put (sm->static_mappings, m);
1158     }
1159
1160   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1161     return 0;
1162
1163   /* Add/delete external address to FIB */
1164   /* *INDENT-OFF* */
1165   pool_foreach (interface, sm->interfaces,
1166   ({
1167     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1168       continue;
1169
1170     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1171     break;
1172   }));
1173   pool_foreach (interface, sm->output_feature_interfaces,
1174   ({
1175     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1176       continue;
1177
1178     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1179     break;
1180   }));
1181   /* *INDENT-ON* */
1182
1183   return 0;
1184 }
1185
1186 int
1187 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1188                                  snat_protocol_t proto,
1189                                  nat44_lb_addr_port_t * locals, u8 is_add,
1190                                  twice_nat_type_t twice_nat, u8 out2in_only,
1191                                  u8 * tag, u32 affinity)
1192 {
1193   snat_main_t *sm = &snat_main;
1194   snat_static_mapping_t *m;
1195   snat_session_key_t m_key;
1196   clib_bihash_kv_8_8_t kv, value;
1197   snat_address_t *a = 0;
1198   int i;
1199   nat44_lb_addr_port_t *local;
1200   u32 elt_index, head_index, ses_index;
1201   snat_main_per_thread_data_t *tsm;
1202   snat_user_key_t u_key;
1203   snat_user_t *u;
1204   snat_session_t *s;
1205   dlist_elt_t *head, *elt;
1206   uword *bitmap = 0;
1207
1208   if (!sm->endpoint_dependent)
1209     return VNET_API_ERROR_FEATURE_DISABLED;
1210
1211   m_key.addr = e_addr;
1212   m_key.port = e_port;
1213   m_key.protocol = proto;
1214   m_key.fib_index = 0;
1215   kv.key = m_key.as_u64;
1216   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1217     m = 0;
1218   else
1219     m = pool_elt_at_index (sm->static_mappings, value.value);
1220
1221   if (is_add)
1222     {
1223       if (m)
1224         return VNET_API_ERROR_VALUE_EXIST;
1225
1226       if (vec_len (locals) < 2)
1227         return VNET_API_ERROR_INVALID_VALUE;
1228
1229       /* Find external address in allocated addresses and reserve port for
1230          address and port pair mapping when dynamic translations enabled */
1231       if (!(sm->static_mapping_only || out2in_only))
1232         {
1233           for (i = 0; i < vec_len (sm->addresses); i++)
1234             {
1235               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1236                 {
1237                   a = sm->addresses + i;
1238                   /* External port must be unused */
1239                   switch (proto)
1240                     {
1241 #define _(N, j, n, s) \
1242                     case SNAT_PROTOCOL_##N: \
1243                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1244                         return VNET_API_ERROR_INVALID_VALUE; \
1245                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1246                       if (e_port > 1024) \
1247                         { \
1248                           a->busy_##n##_ports++; \
1249                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1250                         } \
1251                       break;
1252                       foreach_snat_protocol
1253 #undef _
1254                     default:
1255                       nat_elog_info ("unknown protocol");
1256                       return VNET_API_ERROR_INVALID_VALUE_2;
1257                     }
1258                   break;
1259                 }
1260             }
1261           /* External address must be allocated */
1262           if (!a)
1263             return VNET_API_ERROR_NO_SUCH_ENTRY;
1264         }
1265
1266       pool_get (sm->static_mappings, m);
1267       clib_memset (m, 0, sizeof (*m));
1268       m->tag = vec_dup (tag);
1269       m->external_addr = e_addr;
1270       m->external_port = e_port;
1271       m->proto = proto;
1272       m->twice_nat = twice_nat;
1273       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1274       if (out2in_only)
1275         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1276       m->affinity = affinity;
1277
1278       if (affinity)
1279         m->affinity_per_service_list_head_index =
1280           nat_affinity_get_per_service_list_head_index ();
1281       else
1282         m->affinity_per_service_list_head_index = ~0;
1283
1284       m_key.addr = m->external_addr;
1285       m_key.port = m->external_port;
1286       m_key.protocol = m->proto;
1287       m_key.fib_index = 0;
1288       kv.key = m_key.as_u64;
1289       kv.value = m - sm->static_mappings;
1290       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1291         {
1292           nat_elog_err ("static_mapping_by_external key add failed");
1293           return VNET_API_ERROR_UNSPECIFIED;
1294         }
1295
1296       m_key.fib_index = m->fib_index;
1297       for (i = 0; i < vec_len (locals); i++)
1298         {
1299           locals[i].fib_index =
1300             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1301                                                locals[i].vrf_id,
1302                                                nat_fib_src_low);
1303           m_key.addr = locals[i].addr;
1304           m_key.fib_index = locals[i].fib_index;
1305           if (!out2in_only)
1306             {
1307               m_key.port = locals[i].port;
1308               kv.key = m_key.as_u64;
1309               kv.value = m - sm->static_mappings;
1310               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1311             }
1312           locals[i].prefix = (i == 0) ? locals[i].probability :
1313             (locals[i - 1].prefix + locals[i].probability);
1314           pool_get (m->locals, local);
1315           *local = locals[i];
1316           if (sm->num_workers > 1)
1317             {
1318               ip4_header_t ip = {
1319                 .src_address = locals[i].addr,
1320               };
1321               bitmap =
1322                 clib_bitmap_set (bitmap,
1323                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1324                                  1);
1325             }
1326         }
1327
1328       /* Assign workers */
1329       if (sm->num_workers > 1)
1330         {
1331           /* *INDENT-OFF* */
1332           clib_bitmap_foreach (i, bitmap,
1333             ({
1334                vec_add1(m->workers, i);
1335             }));
1336           /* *INDENT-ON* */
1337         }
1338     }
1339   else
1340     {
1341       if (!m)
1342         return VNET_API_ERROR_NO_SUCH_ENTRY;
1343
1344       if (!is_lb_static_mapping (m))
1345         return VNET_API_ERROR_INVALID_VALUE;
1346
1347       /* Free external address port */
1348       if (!(sm->static_mapping_only || out2in_only))
1349         {
1350           for (i = 0; i < vec_len (sm->addresses); i++)
1351             {
1352               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1353                 {
1354                   a = sm->addresses + i;
1355                   switch (proto)
1356                     {
1357 #define _(N, j, n, s) \
1358                     case SNAT_PROTOCOL_##N: \
1359                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1360                       if (e_port > 1024) \
1361                         { \
1362                           a->busy_##n##_ports--; \
1363                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1364                         } \
1365                       break;
1366                       foreach_snat_protocol
1367 #undef _
1368                     default:
1369                       nat_elog_info ("unknown protocol");
1370                       return VNET_API_ERROR_INVALID_VALUE_2;
1371                     }
1372                   break;
1373                 }
1374             }
1375         }
1376
1377       m_key.addr = m->external_addr;
1378       m_key.port = m->external_port;
1379       m_key.protocol = m->proto;
1380       m_key.fib_index = 0;
1381       kv.key = m_key.as_u64;
1382       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1383         {
1384           nat_elog_err ("static_mapping_by_external key del failed");
1385           return VNET_API_ERROR_UNSPECIFIED;
1386         }
1387
1388       /* *INDENT-OFF* */
1389       pool_foreach (local, m->locals,
1390       ({
1391           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1392                             nat_fib_src_low);
1393           m_key.addr = local->addr;
1394           if (!out2in_only)
1395             {
1396               m_key.port = local->port;
1397               m_key.fib_index = local->fib_index;
1398               kv.key = m_key.as_u64;
1399               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1400                 {
1401                   nat_elog_err ("static_mapping_by_local key del failed");
1402                   return VNET_API_ERROR_UNSPECIFIED;
1403                 }
1404             }
1405
1406           if (sm->num_workers > 1)
1407             {
1408               ip4_header_t ip = {
1409                 .src_address = local->addr,
1410               };
1411               tsm = vec_elt_at_index (sm->per_thread_data,
1412                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1413             }
1414           else
1415             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1416
1417           /* Delete sessions */
1418           u_key.addr = local->addr;
1419           u_key.fib_index = local->fib_index;
1420           kv.key = u_key.as_u64;
1421           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1422             {
1423               u = pool_elt_at_index (tsm->users, value.value);
1424               if (u->nstaticsessions)
1425                 {
1426                   head_index = u->sessions_per_user_list_head_index;
1427                   head = pool_elt_at_index (tsm->list_pool, head_index);
1428                   elt_index = head->next;
1429                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1430                   ses_index = elt->value;
1431                   while (ses_index != ~0)
1432                     {
1433                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1434                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1435                       ses_index = elt->value;
1436
1437                       if (!(is_lb_session (s)))
1438                         continue;
1439
1440                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1441                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1442                         continue;
1443
1444                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1445                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1446                     }
1447                 }
1448             }
1449       }));
1450       /* *INDENT-ON* */
1451       if (m->affinity)
1452         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1453       pool_free (m->locals);
1454       vec_free (m->tag);
1455       vec_free (m->workers);
1456
1457       pool_put (sm->static_mappings, m);
1458     }
1459
1460   return 0;
1461 }
1462
1463 int
1464 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1465                                        ip4_address_t l_addr, u16 l_port,
1466                                        snat_protocol_t proto, u32 vrf_id,
1467                                        u8 probability, u8 is_add)
1468 {
1469   snat_main_t *sm = &snat_main;
1470   snat_static_mapping_t *m = 0;
1471   snat_session_key_t m_key;
1472   clib_bihash_kv_8_8_t kv, value;
1473   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1474   snat_main_per_thread_data_t *tsm;
1475   snat_user_key_t u_key;
1476   snat_user_t *u;
1477   snat_session_t *s;
1478   dlist_elt_t *head, *elt;
1479   u32 elt_index, head_index, ses_index, *locals = 0;
1480   uword *bitmap = 0;
1481   int i;
1482
1483   if (!sm->endpoint_dependent)
1484     return VNET_API_ERROR_FEATURE_DISABLED;
1485
1486   m_key.addr = e_addr;
1487   m_key.port = e_port;
1488   m_key.protocol = proto;
1489   m_key.fib_index = 0;
1490   kv.key = m_key.as_u64;
1491   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1492     m = pool_elt_at_index (sm->static_mappings, value.value);
1493
1494   if (!m)
1495     return VNET_API_ERROR_NO_SUCH_ENTRY;
1496
1497   if (!is_lb_static_mapping (m))
1498     return VNET_API_ERROR_INVALID_VALUE;
1499
1500   /* *INDENT-OFF* */
1501   pool_foreach (local, m->locals,
1502   ({
1503     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1504         (local->vrf_id == vrf_id))
1505       {
1506         match_local = local;
1507         break;
1508       }
1509   }));
1510   /* *INDENT-ON* */
1511
1512   if (is_add)
1513     {
1514       if (match_local)
1515         return VNET_API_ERROR_VALUE_EXIST;
1516
1517       pool_get (m->locals, local);
1518       clib_memset (local, 0, sizeof (*local));
1519       local->addr.as_u32 = l_addr.as_u32;
1520       local->port = l_port;
1521       local->probability = probability;
1522       local->vrf_id = vrf_id;
1523       local->fib_index =
1524         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1525                                            nat_fib_src_low);
1526
1527       if (!is_out2in_only_static_mapping (m))
1528         {
1529           m_key.addr = l_addr;
1530           m_key.port = l_port;
1531           m_key.fib_index = local->fib_index;
1532           kv.key = m_key.as_u64;
1533           kv.value = m - sm->static_mappings;
1534           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1535             nat_elog_err ("static_mapping_by_local key add failed");
1536         }
1537     }
1538   else
1539     {
1540       if (!match_local)
1541         return VNET_API_ERROR_NO_SUCH_ENTRY;
1542
1543       if (pool_elts (m->locals) < 3)
1544         return VNET_API_ERROR_UNSPECIFIED;
1545
1546       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1547                         nat_fib_src_low);
1548
1549       if (!is_out2in_only_static_mapping (m))
1550         {
1551           m_key.addr = l_addr;
1552           m_key.port = l_port;
1553           m_key.fib_index = match_local->fib_index;
1554           kv.key = m_key.as_u64;
1555           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1556             nat_elog_err ("static_mapping_by_local key del failed");
1557         }
1558
1559       if (sm->num_workers > 1)
1560         {
1561           ip4_header_t ip = {
1562             .src_address = local->addr,
1563           };
1564           tsm = vec_elt_at_index (sm->per_thread_data,
1565                                   sm->worker_in2out_cb (&ip, m->fib_index,
1566                                                         0));
1567         }
1568       else
1569         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1570
1571       /* Delete sessions */
1572       u_key.addr = match_local->addr;
1573       u_key.fib_index = match_local->fib_index;
1574       kv.key = u_key.as_u64;
1575       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1576         {
1577           u = pool_elt_at_index (tsm->users, value.value);
1578           if (u->nstaticsessions)
1579             {
1580               head_index = u->sessions_per_user_list_head_index;
1581               head = pool_elt_at_index (tsm->list_pool, head_index);
1582               elt_index = head->next;
1583               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1584               ses_index = elt->value;
1585               while (ses_index != ~0)
1586                 {
1587                   s = pool_elt_at_index (tsm->sessions, ses_index);
1588                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1589                   ses_index = elt->value;
1590
1591                   if (!(is_lb_session (s)))
1592                     continue;
1593
1594                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1595                       (clib_net_to_host_u16 (s->in2out.port) !=
1596                        match_local->port))
1597                     continue;
1598
1599                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1600                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1601                 }
1602             }
1603         }
1604
1605       pool_put (m->locals, match_local);
1606     }
1607
1608   vec_free (m->workers);
1609
1610   /* *INDENT-OFF* */
1611   pool_foreach (local, m->locals,
1612   ({
1613     vec_add1 (locals, local - m->locals);
1614     if (sm->num_workers > 1)
1615       {
1616         ip4_header_t ip;
1617         ip.src_address.as_u32 = local->addr.as_u32,
1618         bitmap = clib_bitmap_set (bitmap,
1619                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1620                                   1);
1621       }
1622   }));
1623   /* *INDENT-ON* */
1624
1625   ASSERT (vec_len (locals) > 1);
1626
1627   local = pool_elt_at_index (m->locals, locals[0]);
1628   local->prefix = local->probability;
1629   for (i = 1; i < vec_len (locals); i++)
1630     {
1631       local = pool_elt_at_index (m->locals, locals[i]);
1632       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1633       local->prefix = local->probability + prev_local->prefix;
1634     }
1635
1636   /* Assign workers */
1637   if (sm->num_workers > 1)
1638     {
1639       /* *INDENT-OFF* */
1640       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1641       /* *INDENT-ON* */
1642     }
1643
1644   return 0;
1645 }
1646
1647 int
1648 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1649                   u8 twice_nat)
1650 {
1651   snat_address_t *a = 0;
1652   snat_session_t *ses;
1653   u32 *ses_to_be_removed = 0, *ses_index;
1654   snat_main_per_thread_data_t *tsm;
1655   snat_static_mapping_t *m;
1656   snat_interface_t *interface;
1657   int i;
1658   snat_address_t *addresses =
1659     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1660
1661   /* Find SNAT address */
1662   for (i = 0; i < vec_len (addresses); i++)
1663     {
1664       if (addresses[i].addr.as_u32 == addr.as_u32)
1665         {
1666           a = addresses + i;
1667           break;
1668         }
1669     }
1670   if (!a)
1671     return VNET_API_ERROR_NO_SUCH_ENTRY;
1672
1673   if (delete_sm)
1674     {
1675       /* *INDENT-OFF* */
1676       pool_foreach (m, sm->static_mappings,
1677       ({
1678           if (m->external_addr.as_u32 == addr.as_u32)
1679             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1680                                             m->local_port, m->external_port,
1681                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1682                                             m->proto, 0, m->twice_nat,
1683                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1684       }));
1685       /* *INDENT-ON* */
1686     }
1687   else
1688     {
1689       /* Check if address is used in some static mapping */
1690       if (is_snat_address_used_in_static_mapping (sm, addr))
1691         {
1692           nat_elog_notice ("address used in static mapping");
1693           return VNET_API_ERROR_UNSPECIFIED;
1694         }
1695     }
1696
1697   if (a->fib_index != ~0)
1698     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1699
1700   /* Delete sessions using address */
1701   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1702     {
1703       /* *INDENT-OFF* */
1704       vec_foreach (tsm, sm->per_thread_data)
1705         {
1706           pool_foreach (ses, tsm->sessions, ({
1707             if (ses->out2in.addr.as_u32 == addr.as_u32)
1708               {
1709                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1710                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1711               }
1712           }));
1713
1714           vec_foreach (ses_index, ses_to_be_removed)
1715             {
1716               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1717               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1718             }
1719
1720           vec_free (ses_to_be_removed);
1721         }
1722       /* *INDENT-ON* */
1723     }
1724
1725 #define _(N, i, n, s) \
1726   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1727   vec_free (a->busy_##n##_ports_per_thread);
1728   foreach_snat_protocol
1729 #undef _
1730     if (twice_nat)
1731     {
1732       vec_del1 (sm->twice_nat_addresses, i);
1733       return 0;
1734     }
1735   else
1736     vec_del1 (sm->addresses, i);
1737
1738   /* Delete external address from FIB */
1739   /* *INDENT-OFF* */
1740   pool_foreach (interface, sm->interfaces,
1741   ({
1742     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1743       continue;
1744
1745     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1746     break;
1747   }));
1748   pool_foreach (interface, sm->output_feature_interfaces,
1749   ({
1750     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1751       continue;
1752
1753     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1754     break;
1755   }));
1756   /* *INDENT-ON* */
1757
1758   return 0;
1759 }
1760
1761 int
1762 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1763 {
1764   snat_main_t *sm = &snat_main;
1765   snat_interface_t *i;
1766   const char *feature_name, *del_feature_name;
1767   snat_address_t *ap;
1768   snat_static_mapping_t *m;
1769   snat_det_map_t *dm;
1770   nat_outside_fib_t *outside_fib;
1771   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1772                                                        sw_if_index);
1773
1774   if (sm->out2in_dpo && !is_inside)
1775     return VNET_API_ERROR_UNSUPPORTED;
1776
1777   /* *INDENT-OFF* */
1778   pool_foreach (i, sm->output_feature_interfaces,
1779   ({
1780     if (i->sw_if_index == sw_if_index)
1781       return VNET_API_ERROR_VALUE_EXIST;
1782   }));
1783   /* *INDENT-ON* */
1784
1785   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1786     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1787   else
1788     {
1789       if (sm->num_workers > 1 && !sm->deterministic)
1790         feature_name =
1791           is_inside ? "nat44-in2out-worker-handoff" :
1792           "nat44-out2in-worker-handoff";
1793       else if (sm->deterministic)
1794         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1795       else if (sm->endpoint_dependent)
1796         {
1797           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1798         }
1799       else
1800         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1801     }
1802
1803   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1804     sm->fq_in2out_index =
1805       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1806
1807   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1808     sm->fq_out2in_index =
1809       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1810
1811   if (!is_inside)
1812     {
1813       /* *INDENT-OFF* */
1814       vec_foreach (outside_fib, sm->outside_fibs)
1815         {
1816           if (outside_fib->fib_index == fib_index)
1817             {
1818               if (is_del)
1819                 {
1820                   outside_fib->refcount--;
1821                   if (!outside_fib->refcount)
1822                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1823                 }
1824               else
1825                 outside_fib->refcount++;
1826               goto feature_set;
1827             }
1828         }
1829       /* *INDENT-ON* */
1830       if (!is_del)
1831         {
1832           vec_add2 (sm->outside_fibs, outside_fib, 1);
1833           outside_fib->refcount = 1;
1834           outside_fib->fib_index = fib_index;
1835         }
1836     }
1837 feature_set:
1838   /* *INDENT-OFF* */
1839   pool_foreach (i, sm->interfaces,
1840   ({
1841     if (i->sw_if_index == sw_if_index)
1842       {
1843         if (is_del)
1844           {
1845             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1846               {
1847                 if (is_inside)
1848                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1849                 else
1850                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1851
1852                 if (sm->num_workers > 1 && !sm->deterministic)
1853                   {
1854                     del_feature_name = "nat44-handoff-classify";
1855                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1856                                                  "nat44-out2in-worker-handoff";
1857                   }
1858                 else if (sm->deterministic)
1859                   {
1860                     del_feature_name = "nat44-det-classify";
1861                     feature_name = !is_inside ?  "nat44-det-in2out" :
1862                                                  "nat44-det-out2in";
1863                   }
1864                 else if (sm->endpoint_dependent)
1865                   {
1866                     del_feature_name = "nat44-ed-classify";
1867                     feature_name = !is_inside ?  "nat-pre-in2out" :
1868                                                  "nat-pre-out2in";
1869                   }
1870                 else
1871                   {
1872                     del_feature_name = "nat44-classify";
1873                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1874                   }
1875
1876                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1877                 if (rv)
1878                   return rv;
1879                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1880                                              sw_if_index, 0, 0, 0);
1881                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1882                                              sw_if_index, 1, 0, 0);
1883                 if (!is_inside)
1884                   {
1885                     if (sm->endpoint_dependent)
1886                       vnet_feature_enable_disable ("ip4-local",
1887                                                    "nat44-ed-hairpinning",
1888                                                    sw_if_index, 1, 0, 0);
1889                     else if (!sm->deterministic)
1890                       vnet_feature_enable_disable ("ip4-local",
1891                                                    "nat44-hairpinning",
1892                                                    sw_if_index, 1, 0, 0);
1893                   }
1894               }
1895             else
1896               {
1897                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1898                 if (rv)
1899                   return rv;
1900                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1901                                              sw_if_index, 0, 0, 0);
1902                 pool_put (sm->interfaces, i);
1903                 if (is_inside)
1904                   {
1905                     if (sm->endpoint_dependent)
1906                       vnet_feature_enable_disable ("ip4-local",
1907                                                    "nat44-ed-hairpinning",
1908                                                    sw_if_index, 0, 0, 0);
1909                     else if (!sm->deterministic)
1910                       vnet_feature_enable_disable ("ip4-local",
1911                                                    "nat44-hairpinning",
1912                                                    sw_if_index, 0, 0, 0);
1913                   }
1914               }
1915           }
1916         else
1917           {
1918             if ((nat_interface_is_inside(i) && is_inside) ||
1919                 (nat_interface_is_outside(i) && !is_inside))
1920               return 0;
1921
1922             if (sm->num_workers > 1 && !sm->deterministic)
1923               {
1924                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1925                                                  "nat44-out2in-worker-handoff";
1926                 feature_name = "nat44-handoff-classify";
1927               }
1928             else if (sm->deterministic)
1929               {
1930                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1931                                                  "nat44-det-out2in";
1932                 feature_name = "nat44-det-classify";
1933               }
1934             else if (sm->endpoint_dependent)
1935               {
1936                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
1937                                                  "nat-pre-out2in";
1938
1939                 feature_name = "nat44-ed-classify";
1940               }
1941             else
1942               {
1943                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1944                 feature_name = "nat44-classify";
1945               }
1946
1947             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1948             if (rv)
1949               return rv;
1950             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1951                                          sw_if_index, 0, 0, 0);
1952             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1953                                          sw_if_index, 1, 0, 0);
1954             if (!is_inside)
1955               {
1956                 if (sm->endpoint_dependent)
1957                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1958                                                sw_if_index, 0, 0, 0);
1959                 else if (!sm->deterministic)
1960                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1961                                                sw_if_index, 0, 0, 0);
1962               }
1963             goto set_flags;
1964           }
1965
1966         goto fib;
1967       }
1968   }));
1969   /* *INDENT-ON* */
1970
1971   if (is_del)
1972     return VNET_API_ERROR_NO_SUCH_ENTRY;
1973
1974   pool_get (sm->interfaces, i);
1975   i->sw_if_index = sw_if_index;
1976   i->flags = 0;
1977   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1978                                0);
1979
1980   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1981   if (rv)
1982     return rv;
1983
1984   if (is_inside && !sm->out2in_dpo)
1985     {
1986       if (sm->endpoint_dependent)
1987         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1988                                      sw_if_index, 1, 0, 0);
1989       else if (!sm->deterministic)
1990         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1991                                      sw_if_index, 1, 0, 0);
1992     }
1993
1994 set_flags:
1995   if (is_inside)
1996     {
1997       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1998       return 0;
1999     }
2000   else
2001     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2002
2003   /* Add/delete external addresses to FIB */
2004 fib:
2005   /* *INDENT-OFF* */
2006   vec_foreach (ap, sm->addresses)
2007     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2008
2009   pool_foreach (m, sm->static_mappings,
2010   ({
2011     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2012       continue;
2013
2014     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2015   }));
2016
2017   pool_foreach (dm, sm->det_maps,
2018   ({
2019     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2020   }));
2021   /* *INDENT-ON* */
2022
2023   return 0;
2024 }
2025
2026 int
2027 snat_interface_add_del_output_feature (u32 sw_if_index,
2028                                        u8 is_inside, int is_del)
2029 {
2030   snat_main_t *sm = &snat_main;
2031   snat_interface_t *i;
2032   snat_address_t *ap;
2033   snat_static_mapping_t *m;
2034   nat_outside_fib_t *outside_fib;
2035   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2036                                                        sw_if_index);
2037
2038
2039   if (sm->deterministic ||
2040       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2041     return VNET_API_ERROR_UNSUPPORTED;
2042
2043   /* *INDENT-OFF* */
2044   pool_foreach (i, sm->interfaces,
2045   ({
2046     if (i->sw_if_index == sw_if_index)
2047       return VNET_API_ERROR_VALUE_EXIST;
2048   }));
2049   /* *INDENT-ON* */
2050
2051   if (!is_inside)
2052     {
2053       /* *INDENT-OFF* */
2054       vec_foreach (outside_fib, sm->outside_fibs)
2055         {
2056           if (outside_fib->fib_index == fib_index)
2057             {
2058               if (is_del)
2059                 {
2060                   outside_fib->refcount--;
2061                   if (!outside_fib->refcount)
2062                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2063                 }
2064               else
2065                 outside_fib->refcount++;
2066               goto feature_set;
2067             }
2068         }
2069       /* *INDENT-ON* */
2070       if (!is_del)
2071         {
2072           vec_add2 (sm->outside_fibs, outside_fib, 1);
2073           outside_fib->refcount = 1;
2074           outside_fib->fib_index = fib_index;
2075         }
2076     }
2077
2078 feature_set:
2079   if (is_inside)
2080     {
2081       if (sm->endpoint_dependent)
2082         {
2083           int rv =
2084             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2085           if (rv)
2086             return rv;
2087           rv =
2088             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2089                                                             !is_del);
2090           if (rv)
2091             return rv;
2092           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2093                                        sw_if_index, !is_del, 0, 0);
2094           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2095                                        sw_if_index, !is_del, 0, 0);
2096         }
2097       else
2098         {
2099           int rv =
2100             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2101           if (rv)
2102             return rv;
2103           rv =
2104             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2105                                                             !is_del);
2106           if (rv)
2107             return rv;
2108           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2109                                        sw_if_index, !is_del, 0, 0);
2110           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2111                                        sw_if_index, !is_del, 0, 0);
2112         }
2113       goto fq;
2114     }
2115
2116   if (sm->num_workers > 1)
2117     {
2118       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2119       if (rv)
2120         return rv;
2121       rv =
2122         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2123       if (rv)
2124         return rv;
2125       vnet_feature_enable_disable ("ip4-unicast",
2126                                    "nat44-out2in-worker-handoff",
2127                                    sw_if_index, !is_del, 0, 0);
2128       vnet_feature_enable_disable ("ip4-output",
2129                                    "nat44-in2out-output-worker-handoff",
2130                                    sw_if_index, !is_del, 0, 0);
2131     }
2132   else
2133     {
2134       if (sm->endpoint_dependent)
2135         {
2136           int rv =
2137             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2138           if (rv)
2139             return rv;
2140           rv =
2141             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2142                                                             !is_del);
2143           if (rv)
2144             return rv;
2145           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2146                                        sw_if_index, !is_del, 0, 0);
2147           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2148                                        sw_if_index, !is_del, 0, 0);
2149         }
2150       else
2151         {
2152           int rv =
2153             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2154           if (rv)
2155             return rv;
2156           rv =
2157             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2158                                                             !is_del);
2159           if (rv)
2160             return rv;
2161           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2162                                        sw_if_index, !is_del, 0, 0);
2163           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2164                                        sw_if_index, !is_del, 0, 0);
2165         }
2166     }
2167
2168 fq:
2169   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2170     sm->fq_in2out_output_index =
2171       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2172
2173   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2174     sm->fq_out2in_index =
2175       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2176
2177   /* *INDENT-OFF* */
2178   pool_foreach (i, sm->output_feature_interfaces,
2179   ({
2180     if (i->sw_if_index == sw_if_index)
2181       {
2182         if (is_del)
2183           pool_put (sm->output_feature_interfaces, i);
2184         else
2185           return VNET_API_ERROR_VALUE_EXIST;
2186
2187         goto fib;
2188       }
2189   }));
2190   /* *INDENT-ON* */
2191
2192   if (is_del)
2193     return VNET_API_ERROR_NO_SUCH_ENTRY;
2194
2195   pool_get (sm->output_feature_interfaces, i);
2196   i->sw_if_index = sw_if_index;
2197   i->flags = 0;
2198   if (is_inside)
2199     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2200   else
2201     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2202
2203   /* Add/delete external addresses to FIB */
2204 fib:
2205   if (is_inside)
2206     return 0;
2207
2208   /* *INDENT-OFF* */
2209   vec_foreach (ap, sm->addresses)
2210     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2211
2212   pool_foreach (m, sm->static_mappings,
2213   ({
2214     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2215       continue;
2216
2217     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2218   }));
2219   /* *INDENT-ON* */
2220
2221   return 0;
2222 }
2223
2224 int
2225 snat_set_workers (uword * bitmap)
2226 {
2227   snat_main_t *sm = &snat_main;
2228   int i, j = 0;
2229
2230   if (sm->num_workers < 2)
2231     return VNET_API_ERROR_FEATURE_DISABLED;
2232
2233   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2234     return VNET_API_ERROR_INVALID_WORKER;
2235
2236   vec_free (sm->workers);
2237   /* *INDENT-OFF* */
2238   clib_bitmap_foreach (i, bitmap,
2239     ({
2240       vec_add1(sm->workers, i);
2241       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2242       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2243       j++;
2244     }));
2245   /* *INDENT-ON* */
2246
2247   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2248   sm->num_snat_thread = _vec_len (sm->workers);
2249
2250   return 0;
2251 }
2252
2253 static void
2254 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2255                          u32 old_fib_index)
2256 {
2257   snat_main_t *sm = &snat_main;
2258   nat_outside_fib_t *outside_fib;
2259   snat_interface_t *i;
2260   u8 is_add = 1;
2261   u8 match = 0;
2262
2263   if (new_fib_index == old_fib_index)
2264     return;
2265
2266   if (!vec_len (sm->outside_fibs))
2267     return;
2268
2269   /* *INDENT-OFF* */
2270   pool_foreach (i, sm->interfaces,
2271     ({
2272       if (i->sw_if_index == sw_if_index)
2273         {
2274           if (!(nat_interface_is_outside (i)))
2275             return;
2276           match = 1;
2277         }
2278     }));
2279
2280   pool_foreach (i, sm->output_feature_interfaces,
2281     ({
2282       if (i->sw_if_index == sw_if_index)
2283         {
2284           if (!(nat_interface_is_outside (i)))
2285             return;
2286           match = 1;
2287         }
2288     }));
2289   /* *INDENT-ON* */
2290
2291   if (!match)
2292     return;
2293
2294   vec_foreach (outside_fib, sm->outside_fibs)
2295   {
2296     if (outside_fib->fib_index == old_fib_index)
2297       {
2298         outside_fib->refcount--;
2299         if (!outside_fib->refcount)
2300           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2301         break;
2302       }
2303   }
2304
2305   vec_foreach (outside_fib, sm->outside_fibs)
2306   {
2307     if (outside_fib->fib_index == new_fib_index)
2308       {
2309         outside_fib->refcount++;
2310         is_add = 0;
2311         break;
2312       }
2313   }
2314
2315   if (is_add)
2316     {
2317       vec_add2 (sm->outside_fibs, outside_fib, 1);
2318       outside_fib->refcount = 1;
2319       outside_fib->fib_index = new_fib_index;
2320     }
2321 }
2322
2323 static void
2324 snat_ip4_table_bind (ip4_main_t * im,
2325                      uword opaque,
2326                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2327 {
2328   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2329 }
2330
2331 static void
2332 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2333                                        uword opaque,
2334                                        u32 sw_if_index,
2335                                        ip4_address_t * address,
2336                                        u32 address_length,
2337                                        u32 if_address_index, u32 is_delete);
2338
2339 static void
2340 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2341                                  uword opaque,
2342                                  u32 sw_if_index,
2343                                  ip4_address_t * address,
2344                                  u32 address_length,
2345                                  u32 if_address_index, u32 is_delete);
2346
2347 static int
2348 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2349                                  u32 fib_index,
2350                                  u32 thread_index,
2351                                  snat_session_key_t * k,
2352                                  u16 port_per_thread, u32 snat_thread_index);
2353
2354 static clib_error_t *
2355 snat_init (vlib_main_t * vm)
2356 {
2357   snat_main_t *sm = &snat_main;
2358   clib_error_t *error = 0;
2359   ip4_main_t *im = &ip4_main;
2360   ip_lookup_main_t *lm = &im->lookup_main;
2361   uword *p;
2362   vlib_thread_registration_t *tr;
2363   vlib_thread_main_t *tm = vlib_get_thread_main ();
2364   uword *bitmap = 0;
2365   u32 i;
2366   ip4_add_del_interface_address_callback_t cb4;
2367   vlib_node_t *node;
2368
2369   sm->vlib_main = vm;
2370   sm->vnet_main = vnet_get_main ();
2371   sm->ip4_main = im;
2372   sm->ip4_lookup_main = lm;
2373   sm->api_main = vlibapi_get_main ();
2374   sm->first_worker_index = 0;
2375   sm->num_workers = 0;
2376   sm->num_snat_thread = 1;
2377   sm->workers = 0;
2378   sm->port_per_thread = 0xffff - 1024;
2379   sm->fq_in2out_index = ~0;
2380   sm->fq_in2out_output_index = ~0;
2381   sm->fq_out2in_index = ~0;
2382
2383
2384   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2385   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2386   sm->forwarding_enabled = 0;
2387   sm->log_class = vlib_log_register_class ("nat", 0);
2388   sm->log_level = SNAT_LOG_ERROR;
2389   sm->mss_clamping = 0;
2390
2391   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2392   sm->error_node_index = node->index;
2393
2394   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2395   sm->pre_in2out_node_index = node->index;
2396   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2397   sm->pre_out2in_node_index = node->index;
2398
2399   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2400   sm->pre_in2out_node_index = node->index;
2401
2402   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2403   sm->pre_out2in_node_index = node->index;
2404
2405   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2406   sm->in2out_node_index = node->index;
2407   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2408   sm->in2out_output_node_index = node->index;
2409   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2410   sm->in2out_fast_node_index = node->index;
2411   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2412   sm->in2out_slowpath_node_index = node->index;
2413   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2414   sm->in2out_slowpath_output_node_index = node->index;
2415
2416   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2417   sm->ed_in2out_node_index = node->index;
2418   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2419   sm->ed_in2out_slowpath_node_index = node->index;
2420
2421   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2422   sm->out2in_node_index = node->index;
2423   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2424   sm->out2in_fast_node_index = node->index;
2425
2426   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2427   sm->ed_out2in_node_index = node->index;
2428   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2429   sm->ed_out2in_slowpath_node_index = node->index;
2430
2431   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2432   sm->det_in2out_node_index = node->index;
2433   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2434   sm->det_out2in_node_index = node->index;
2435
2436   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2437   sm->hairpinning_node_index = node->index;
2438   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2439   sm->hairpin_dst_node_index = node->index;
2440   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2441   sm->hairpin_src_node_index = node->index;
2442   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2443   sm->ed_hairpinning_node_index = node->index;
2444   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2445   sm->ed_hairpin_dst_node_index = node->index;
2446   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2447   sm->ed_hairpin_src_node_index = node->index;
2448
2449   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2450   if (p)
2451     {
2452       tr = (vlib_thread_registration_t *) p[0];
2453       if (tr)
2454         {
2455           sm->num_workers = tr->count;
2456           sm->first_worker_index = tr->first_index;
2457         }
2458     }
2459
2460   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2461
2462   /* Use all available workers by default */
2463   if (sm->num_workers > 1)
2464     {
2465       for (i = 0; i < sm->num_workers; i++)
2466         bitmap = clib_bitmap_set (bitmap, i, 1);
2467       snat_set_workers (bitmap);
2468       clib_bitmap_free (bitmap);
2469     }
2470   else
2471     {
2472       sm->per_thread_data[0].snat_thread_index = 0;
2473     }
2474
2475   error = snat_api_init (vm, sm);
2476   if (error)
2477     return error;
2478
2479   /* Set up the interface address add/del callback */
2480   cb4.function = snat_ip4_add_del_interface_address_cb;
2481   cb4.function_opaque = 0;
2482
2483   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2484
2485   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2486   cb4.function_opaque = 0;
2487
2488   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2489
2490   nat_dpo_module_init ();
2491
2492   /* Init counters */
2493   sm->total_users.name = "total-users";
2494   sm->total_users.stat_segment_name = "/nat44/total-users";
2495   vlib_validate_simple_counter (&sm->total_users, 0);
2496   vlib_zero_simple_counter (&sm->total_users, 0);
2497   sm->total_sessions.name = "total-sessions";
2498   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2499   vlib_validate_simple_counter (&sm->total_sessions, 0);
2500   vlib_zero_simple_counter (&sm->total_sessions, 0);
2501
2502   /* Init IPFIX logging */
2503   snat_ipfix_logging_init (vm);
2504
2505   /* Init NAT64 */
2506   error = nat64_init (vm);
2507   if (error)
2508     return error;
2509
2510   dslite_init (vm);
2511
2512   nat66_init (vm);
2513
2514   ip4_table_bind_callback_t cbt4 = {
2515     .function = snat_ip4_table_bind,
2516   };
2517   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2518
2519   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2520                                         FIB_SOURCE_PRIORITY_HI,
2521                                         FIB_SOURCE_BH_SIMPLE);
2522   nat_fib_src_low = fib_source_allocate ("nat-low",
2523                                          FIB_SOURCE_PRIORITY_LOW,
2524                                          FIB_SOURCE_BH_SIMPLE);
2525
2526   return error;
2527 }
2528
2529 VLIB_INIT_FUNCTION (snat_init);
2530
2531 void
2532 snat_free_outside_address_and_port (snat_address_t * addresses,
2533                                     u32 thread_index, snat_session_key_t * k)
2534 {
2535   snat_address_t *a;
2536   u32 address_index;
2537   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2538
2539   for (address_index = 0; address_index < vec_len (addresses);
2540        address_index++)
2541     {
2542       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2543         break;
2544     }
2545
2546   ASSERT (address_index < vec_len (addresses));
2547
2548   a = addresses + address_index;
2549
2550   switch (k->protocol)
2551     {
2552 #define _(N, i, n, s) \
2553     case SNAT_PROTOCOL_##N: \
2554       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2555         port_host_byte_order) == 1); \
2556       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2557         port_host_byte_order, 0); \
2558       a->busy_##n##_ports--; \
2559       a->busy_##n##_ports_per_thread[thread_index]--; \
2560       break;
2561       foreach_snat_protocol
2562 #undef _
2563     default:
2564       nat_elog_info ("unknown protocol");
2565       return;
2566     }
2567 }
2568
2569 static int
2570 nat_set_outside_address_and_port (snat_address_t * addresses,
2571                                   u32 thread_index, snat_session_key_t * k)
2572 {
2573   snat_address_t *a = 0;
2574   u32 address_index;
2575   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2576
2577   for (address_index = 0; address_index < vec_len (addresses);
2578        address_index++)
2579     {
2580       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2581         continue;
2582
2583       a = addresses + address_index;
2584       switch (k->protocol)
2585         {
2586 #define _(N, j, n, s) \
2587         case SNAT_PROTOCOL_##N: \
2588           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2589             return VNET_API_ERROR_INSTANCE_IN_USE; \
2590           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2591           a->busy_##n##_ports_per_thread[thread_index]++; \
2592           a->busy_##n##_ports++; \
2593           return 0;
2594           foreach_snat_protocol
2595 #undef _
2596         default:
2597           nat_elog_info ("unknown protocol");
2598           return 1;
2599         }
2600     }
2601
2602   return VNET_API_ERROR_NO_SUCH_ENTRY;
2603 }
2604
2605 int
2606 snat_static_mapping_match (snat_main_t * sm,
2607                            snat_session_key_t match,
2608                            snat_session_key_t * mapping,
2609                            u8 by_external,
2610                            u8 * is_addr_only,
2611                            twice_nat_type_t * twice_nat,
2612                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2613                            u8 * is_identity_nat)
2614 {
2615   clib_bihash_kv_8_8_t kv, value;
2616   snat_static_mapping_t *m;
2617   snat_session_key_t m_key;
2618   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2619   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2620   u8 backend_index;
2621   nat44_lb_addr_port_t *local;
2622
2623   m_key.fib_index = match.fib_index;
2624   if (by_external)
2625     {
2626       mapping_hash = &sm->static_mapping_by_external;
2627       m_key.fib_index = 0;
2628     }
2629
2630   m_key.addr = match.addr;
2631   m_key.port = clib_net_to_host_u16 (match.port);
2632   m_key.protocol = match.protocol;
2633
2634   kv.key = m_key.as_u64;
2635
2636   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2637     {
2638       /* Try address only mapping */
2639       m_key.port = 0;
2640       m_key.protocol = 0;
2641       kv.key = m_key.as_u64;
2642       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2643         return 1;
2644     }
2645
2646   m = pool_elt_at_index (sm->static_mappings, value.value);
2647
2648   if (by_external)
2649     {
2650       if (is_lb_static_mapping (m))
2651         {
2652           if (PREDICT_FALSE (lb != 0))
2653             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2654           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2655                                                           match.addr,
2656                                                           match.protocol,
2657                                                           match.port,
2658                                                           &backend_index))
2659             {
2660               local = pool_elt_at_index (m->locals, backend_index);
2661               mapping->addr = local->addr;
2662               mapping->port = clib_host_to_net_u16 (local->port);
2663               mapping->fib_index = local->fib_index;
2664               goto end;
2665             }
2666           // pick locals matching this worker
2667           if (PREDICT_FALSE (sm->num_workers > 1))
2668             {
2669               u32 thread_index = vlib_get_thread_index ();
2670               /* *INDENT-OFF* */
2671               pool_foreach_index (i, m->locals,
2672               ({
2673                 local = pool_elt_at_index (m->locals, i);
2674
2675                 ip4_header_t ip = {
2676                   .src_address = local->addr,
2677                 };
2678
2679                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2680                     thread_index)
2681                   {
2682                     vec_add1 (tmp, i);
2683                   }
2684               }));
2685               /* *INDENT-ON* */
2686               ASSERT (vec_len (tmp) != 0);
2687             }
2688           else
2689             {
2690               /* *INDENT-OFF* */
2691               pool_foreach_index (i, m->locals,
2692               ({
2693                 vec_add1 (tmp, i);
2694               }));
2695               /* *INDENT-ON* */
2696             }
2697           hi = vec_len (tmp) - 1;
2698           local = pool_elt_at_index (m->locals, tmp[hi]);
2699           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2700           while (lo < hi)
2701             {
2702               mid = ((hi - lo) >> 1) + lo;
2703               local = pool_elt_at_index (m->locals, tmp[mid]);
2704               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2705             }
2706           local = pool_elt_at_index (m->locals, tmp[lo]);
2707           if (!(local->prefix >= rand))
2708             return 1;
2709           mapping->addr = local->addr;
2710           mapping->port = clib_host_to_net_u16 (local->port);
2711           mapping->fib_index = local->fib_index;
2712           if (m->affinity)
2713             {
2714               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2715                                                 match.protocol, match.port,
2716                                                 tmp[lo], m->affinity,
2717                                                 m->affinity_per_service_list_head_index))
2718                 nat_elog_info ("create affinity record failed");
2719             }
2720           vec_free (tmp);
2721         }
2722       else
2723         {
2724           if (PREDICT_FALSE (lb != 0))
2725             *lb = NO_LB_NAT;
2726           mapping->fib_index = m->fib_index;
2727           mapping->addr = m->local_addr;
2728           /* Address only mapping doesn't change port */
2729           mapping->port = is_addr_only_static_mapping (m) ? match.port
2730             : clib_host_to_net_u16 (m->local_port);
2731         }
2732       mapping->protocol = m->proto;
2733     }
2734   else
2735     {
2736       mapping->addr = m->external_addr;
2737       /* Address only mapping doesn't change port */
2738       mapping->port = is_addr_only_static_mapping (m) ? match.port
2739         : clib_host_to_net_u16 (m->external_port);
2740       mapping->fib_index = sm->outside_fib_index;
2741     }
2742
2743 end:
2744   if (PREDICT_FALSE (is_addr_only != 0))
2745     *is_addr_only = is_addr_only_static_mapping (m);
2746
2747   if (PREDICT_FALSE (twice_nat != 0))
2748     *twice_nat = m->twice_nat;
2749
2750   if (PREDICT_FALSE (is_identity_nat != 0))
2751     *is_identity_nat = is_identity_static_mapping (m);
2752
2753   return 0;
2754 }
2755
2756 static_always_inline u16
2757 snat_random_port (u16 min, u16 max)
2758 {
2759   snat_main_t *sm = &snat_main;
2760   return min + random_u32 (&sm->random_seed) /
2761     (random_u32_max () / (max - min + 1) + 1);
2762 }
2763
2764 int
2765 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2766                                      u32 fib_index,
2767                                      u32 thread_index,
2768                                      snat_session_key_t * k,
2769                                      u16 port_per_thread,
2770                                      u32 snat_thread_index)
2771 {
2772   snat_main_t *sm = &snat_main;
2773
2774   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2775                                   port_per_thread, snat_thread_index);
2776 }
2777
2778 static int
2779 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2780                                  u32 fib_index,
2781                                  u32 thread_index,
2782                                  snat_session_key_t * k,
2783                                  u16 port_per_thread, u32 snat_thread_index)
2784 {
2785   int i;
2786   snat_address_t *a, *ga = 0;
2787   u32 portnum;
2788
2789   for (i = 0; i < vec_len (addresses); i++)
2790     {
2791       a = addresses + i;
2792       switch (k->protocol)
2793         {
2794 #define _(N, j, n, s) \
2795         case SNAT_PROTOCOL_##N: \
2796           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2797             { \
2798               if (a->fib_index == fib_index) \
2799                 { \
2800                   while (1) \
2801                     { \
2802                       portnum = (port_per_thread * \
2803                         snat_thread_index) + \
2804                         snat_random_port(1, port_per_thread) + 1024; \
2805                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2806                         continue; \
2807                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2808                       a->busy_##n##_ports_per_thread[thread_index]++; \
2809                       a->busy_##n##_ports++; \
2810                       k->addr = a->addr; \
2811                       k->port = clib_host_to_net_u16(portnum); \
2812                       return 0; \
2813                     } \
2814                 } \
2815               else if (a->fib_index == ~0) \
2816                 { \
2817                   ga = a; \
2818                 } \
2819             } \
2820           break;
2821           foreach_snat_protocol
2822 #undef _
2823         default:
2824           nat_elog_info ("unknown protocol");
2825           return 1;
2826         }
2827
2828     }
2829
2830   if (ga)
2831     {
2832       a = ga;
2833       switch (k->protocol)
2834         {
2835 #define _(N, j, n, s) \
2836         case SNAT_PROTOCOL_##N: \
2837           while (1) \
2838             { \
2839               portnum = (port_per_thread * \
2840                 snat_thread_index) + \
2841                 snat_random_port(1, port_per_thread) + 1024; \
2842               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2843                 continue; \
2844               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2845               a->busy_##n##_ports_per_thread[thread_index]++; \
2846               a->busy_##n##_ports++; \
2847               k->addr = a->addr; \
2848               k->port = clib_host_to_net_u16(portnum); \
2849               return 0; \
2850             }
2851           break;
2852           foreach_snat_protocol
2853 #undef _
2854         default:
2855           nat_elog_info ("unknown protocol");
2856           return 1;
2857         }
2858     }
2859
2860   /* Totally out of translations to use... */
2861   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2862   return 1;
2863 }
2864
2865 static int
2866 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2867                               u32 fib_index,
2868                               u32 thread_index,
2869                               snat_session_key_t * k,
2870                               u16 port_per_thread, u32 snat_thread_index)
2871 {
2872   snat_main_t *sm = &snat_main;
2873   snat_address_t *a = addresses;
2874   u16 m, ports, portnum, A, j;
2875   m = 16 - (sm->psid_offset + sm->psid_length);
2876   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2877
2878   if (!vec_len (addresses))
2879     goto exhausted;
2880
2881   switch (k->protocol)
2882     {
2883 #define _(N, i, n, s) \
2884     case SNAT_PROTOCOL_##N: \
2885       if (a->busy_##n##_ports < ports) \
2886         { \
2887           while (1) \
2888             { \
2889               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2890               j = snat_random_port(0, pow2_mask(m)); \
2891               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2892               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2893                 continue; \
2894               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2895               a->busy_##n##_ports++; \
2896               k->addr = a->addr; \
2897               k->port = clib_host_to_net_u16 (portnum); \
2898               return 0; \
2899             } \
2900         } \
2901       break;
2902       foreach_snat_protocol
2903 #undef _
2904     default:
2905       nat_elog_info ("unknown protocol");
2906       return 1;
2907     }
2908
2909 exhausted:
2910   /* Totally out of translations to use... */
2911   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2912   return 1;
2913 }
2914
2915 static int
2916 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2917                                u32 fib_index,
2918                                u32 thread_index,
2919                                snat_session_key_t * k,
2920                                u16 port_per_thread, u32 snat_thread_index)
2921 {
2922   snat_main_t *sm = &snat_main;
2923   snat_address_t *a = addresses;
2924   u16 portnum, ports;
2925
2926   ports = sm->end_port - sm->start_port + 1;
2927
2928   if (!vec_len (addresses))
2929     goto exhausted;
2930
2931   switch (k->protocol)
2932     {
2933 #define _(N, i, n, s) \
2934     case SNAT_PROTOCOL_##N: \
2935       if (a->busy_##n##_ports < ports) \
2936         { \
2937           while (1) \
2938             { \
2939               portnum = snat_random_port(sm->start_port, sm->end_port); \
2940               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2941                 continue; \
2942               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2943               a->busy_##n##_ports++; \
2944               k->addr = a->addr; \
2945               k->port = clib_host_to_net_u16 (portnum); \
2946               return 0; \
2947             } \
2948         } \
2949       break;
2950       foreach_snat_protocol
2951 #undef _
2952     default:
2953       nat_elog_info ("unknown protocol");
2954       return 1;
2955     }
2956
2957 exhausted:
2958   /* Totally out of translations to use... */
2959   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2960   return 1;
2961 }
2962
2963 void
2964 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2965 {
2966   dpo_id_t dpo_v4 = DPO_INVALID;
2967   fib_prefix_t pfx = {
2968     .fp_proto = FIB_PROTOCOL_IP4,
2969     .fp_len = 32,
2970     .fp_addr.ip4.as_u32 = addr.as_u32,
2971   };
2972
2973   if (is_add)
2974     {
2975       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2976       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
2977                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2978       dpo_reset (&dpo_v4);
2979     }
2980   else
2981     {
2982       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
2983     }
2984 }
2985
2986 u8 *
2987 format_session_kvp (u8 * s, va_list * args)
2988 {
2989   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2990   snat_session_key_t k;
2991
2992   k.as_u64 = v->key;
2993
2994   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2995
2996   return s;
2997 }
2998
2999 u8 *
3000 format_static_mapping_kvp (u8 * s, va_list * args)
3001 {
3002   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3003   snat_session_key_t k;
3004
3005   k.as_u64 = v->key;
3006
3007   s = format (s, "%U static-mapping-index %llu",
3008               format_static_mapping_key, &k, v->value);
3009
3010   return s;
3011 }
3012
3013 u8 *
3014 format_user_kvp (u8 * s, va_list * args)
3015 {
3016   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3017   snat_user_key_t k;
3018
3019   k.as_u64 = v->key;
3020
3021   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3022               k.fib_index, v->value);
3023
3024   return s;
3025 }
3026
3027 u8 *
3028 format_ed_session_kvp (u8 * s, va_list * args)
3029 {
3030   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3031   nat_ed_ses_key_t k;
3032
3033   k.as_u64[0] = v->key[0];
3034   k.as_u64[1] = v->key[1];
3035
3036   s =
3037     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3038             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
3039             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
3040             format_ip_protocol, k.proto, k.fib_index, v->value);
3041
3042   return s;
3043 }
3044
3045 static u32
3046 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3047                            u8 is_output)
3048 {
3049   snat_main_t *sm = &snat_main;
3050   u32 next_worker_index = 0;
3051   u32 hash;
3052
3053   next_worker_index = sm->first_worker_index;
3054   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3055     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3056
3057   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3058     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3059   else
3060     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3061
3062   return next_worker_index;
3063 }
3064
3065 static u32
3066 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3067                            u32 rx_fib_index0, u8 is_output)
3068 {
3069   snat_main_t *sm = &snat_main;
3070   udp_header_t *udp;
3071   u16 port;
3072   snat_session_key_t m_key;
3073   clib_bihash_kv_8_8_t kv, value;
3074   snat_static_mapping_t *m;
3075   u32 proto;
3076   u32 next_worker_index = 0;
3077
3078   /* first try static mappings without port */
3079   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3080     {
3081       m_key.addr = ip0->dst_address;
3082       m_key.port = 0;
3083       m_key.protocol = 0;
3084       m_key.fib_index = rx_fib_index0;
3085       kv.key = m_key.as_u64;
3086       if (!clib_bihash_search_8_8
3087           (&sm->static_mapping_by_external, &kv, &value))
3088         {
3089           m = pool_elt_at_index (sm->static_mappings, value.value);
3090           return m->workers[0];
3091         }
3092     }
3093
3094   proto = ip_proto_to_snat_proto (ip0->protocol);
3095   udp = ip4_next_header (ip0);
3096   port = udp->dst_port;
3097
3098   /* unknown protocol */
3099   if (PREDICT_FALSE (proto == ~0))
3100     {
3101       /* use current thread */
3102       return vlib_get_thread_index ();
3103     }
3104
3105   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3106     {
3107       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3108       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3109       if (!icmp_type_is_error_message
3110           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3111         port = vnet_buffer (b)->ip.reass.l4_src_port;
3112       else
3113         {
3114           /* if error message, then it's not fragmented and we can access it */
3115           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3116           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3117           void *l4_header = ip4_next_header (inner_ip);
3118           switch (proto)
3119             {
3120             case SNAT_PROTOCOL_ICMP:
3121               icmp = (icmp46_header_t *) l4_header;
3122               echo = (icmp_echo_header_t *) (icmp + 1);
3123               port = echo->identifier;
3124               break;
3125             case SNAT_PROTOCOL_UDP:
3126             case SNAT_PROTOCOL_TCP:
3127               port = ((tcp_udp_header_t *) l4_header)->src_port;
3128               break;
3129             default:
3130               return vlib_get_thread_index ();
3131             }
3132         }
3133     }
3134
3135   /* try static mappings with port */
3136   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3137     {
3138       m_key.addr = ip0->dst_address;
3139       m_key.port = clib_net_to_host_u16 (port);
3140       m_key.protocol = proto;
3141       m_key.fib_index = rx_fib_index0;
3142       kv.key = m_key.as_u64;
3143       if (!clib_bihash_search_8_8
3144           (&sm->static_mapping_by_external, &kv, &value))
3145         {
3146           m = pool_elt_at_index (sm->static_mappings, value.value);
3147           return m->workers[0];
3148         }
3149     }
3150
3151   /* worker by outside port */
3152   next_worker_index = sm->first_worker_index;
3153   next_worker_index +=
3154     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3155   return next_worker_index;
3156 }
3157
3158 static u32
3159 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3160                                u8 is_output)
3161 {
3162   snat_main_t *sm = &snat_main;
3163   u32 next_worker_index = sm->first_worker_index;
3164   u32 hash;
3165
3166   clib_bihash_kv_16_8_t kv16, value16;
3167   snat_main_per_thread_data_t *tsm;
3168   udp_header_t *udp;
3169
3170   if (PREDICT_FALSE (is_output))
3171     {
3172       u32 fib_index = sm->outside_fib_index;
3173       nat_outside_fib_t *outside_fib;
3174       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3175       fib_prefix_t pfx = {
3176         .fp_proto = FIB_PROTOCOL_IP4,
3177         .fp_len = 32,
3178         .fp_addr = {
3179                     .ip4.as_u32 = ip->dst_address.as_u32,
3180                     }
3181         ,
3182       };
3183
3184       udp = ip4_next_header (ip);
3185
3186       switch (vec_len (sm->outside_fibs))
3187         {
3188         case 0:
3189           fib_index = sm->outside_fib_index;
3190           break;
3191         case 1:
3192           fib_index = sm->outside_fibs[0].fib_index;
3193           break;
3194         default:
3195             /* *INDENT-OFF* */
3196             vec_foreach (outside_fib, sm->outside_fibs)
3197               {
3198                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3199                 if (FIB_NODE_INDEX_INVALID != fei)
3200                   {
3201                     if (fib_entry_get_resolving_interface (fei) != ~0)
3202                       {
3203                         fib_index = outside_fib->fib_index;
3204                         break;
3205                       }
3206                   }
3207               }
3208             /* *INDENT-ON* */
3209           break;
3210         }
3211
3212       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3213                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3214
3215       /* *INDENT-OFF* */
3216       vec_foreach (tsm, sm->per_thread_data)
3217         {
3218           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3219                                                       &kv16, &value16)))
3220             {
3221               next_worker_index += tsm->thread_index;
3222
3223               nat_elog_debug_handoff (
3224                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3225                 next_worker_index, fib_index,
3226                 clib_net_to_host_u32 (ip->src_address.as_u32),
3227                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3228
3229               return next_worker_index;
3230             }
3231         }
3232       /* *INDENT-ON* */
3233     }
3234
3235   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3236     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3237
3238   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3239     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3240   else
3241     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3242
3243   if (PREDICT_TRUE (!is_output))
3244     {
3245       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3246                               next_worker_index, rx_fib_index,
3247                               clib_net_to_host_u32 (ip->src_address.as_u32),
3248                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3249     }
3250   else
3251     {
3252       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3253                               next_worker_index, rx_fib_index,
3254                               clib_net_to_host_u32 (ip->src_address.as_u32),
3255                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3256     }
3257
3258   return next_worker_index;
3259 }
3260
3261 static u32
3262 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3263                                u32 rx_fib_index, u8 is_output)
3264 {
3265   snat_main_t *sm = &snat_main;
3266   clib_bihash_kv_8_8_t kv, value;
3267   clib_bihash_kv_16_8_t kv16, value16;
3268   snat_main_per_thread_data_t *tsm;
3269
3270   u32 proto, next_worker_index = 0;
3271   udp_header_t *udp;
3272   u16 port;
3273   snat_static_mapping_t *m;
3274   u32 hash;
3275
3276   proto = ip_proto_to_snat_proto (ip->protocol);
3277
3278   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3279     {
3280       udp = ip4_next_header (ip);
3281
3282       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3283                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3284
3285       /* *INDENT-OFF* */
3286       vec_foreach (tsm, sm->per_thread_data)
3287         {
3288           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3289                                                       &kv16, &value16)))
3290             {
3291               next_worker_index = sm->first_worker_index + tsm->thread_index;
3292               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3293                           next_worker_index, rx_fib_index,
3294                           clib_net_to_host_u32 (ip->src_address.as_u32),
3295                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3296               return next_worker_index;
3297             }
3298           }
3299         /* *INDENT-ON* */
3300     }
3301   else if (proto == SNAT_PROTOCOL_ICMP)
3302     {
3303       nat_ed_ses_key_t key;
3304
3305       if (!get_icmp_o2i_ed_key (b, ip, &key))
3306         {
3307
3308           key.fib_index = rx_fib_index;
3309           kv16.key[0] = key.as_u64[0];
3310           kv16.key[1] = key.as_u64[1];
3311
3312           /* *INDENT-OFF* */
3313           vec_foreach (tsm, sm->per_thread_data)
3314             {
3315               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3316                                                           &kv16, &value16)))
3317                 {
3318                   next_worker_index = sm->first_worker_index +
3319                                       tsm->thread_index;
3320                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3321                               next_worker_index, rx_fib_index,
3322                               clib_net_to_host_u32 (ip->src_address.as_u32),
3323                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3324                   return next_worker_index;
3325                 }
3326             }
3327           /* *INDENT-ON* */
3328         }
3329     }
3330
3331   /* first try static mappings without port */
3332   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3333     {
3334       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3335       if (!clib_bihash_search_8_8
3336           (&sm->static_mapping_by_external, &kv, &value))
3337         {
3338           m = pool_elt_at_index (sm->static_mappings, value.value);
3339           next_worker_index = m->workers[0];
3340           goto done;
3341         }
3342     }
3343
3344   /* unknown protocol */
3345   if (PREDICT_FALSE (proto == ~0))
3346     {
3347       /* use current thread */
3348       next_worker_index = vlib_get_thread_index ();
3349       goto done;
3350     }
3351
3352   udp = ip4_next_header (ip);
3353   port = udp->dst_port;
3354
3355   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3356     {
3357       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3358       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3359       if (!icmp_type_is_error_message
3360           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3361         port = vnet_buffer (b)->ip.reass.l4_src_port;
3362       else
3363         {
3364           /* if error message, then it's not fragmented and we can access it */
3365           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3366           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3367           void *l4_header = ip4_next_header (inner_ip);
3368           switch (proto)
3369             {
3370             case SNAT_PROTOCOL_ICMP:
3371               icmp = (icmp46_header_t *) l4_header;
3372               echo = (icmp_echo_header_t *) (icmp + 1);
3373               port = echo->identifier;
3374               break;
3375             case SNAT_PROTOCOL_UDP:
3376             case SNAT_PROTOCOL_TCP:
3377               port = ((tcp_udp_header_t *) l4_header)->src_port;
3378               break;
3379             default:
3380               next_worker_index = vlib_get_thread_index ();
3381               goto done;
3382             }
3383         }
3384     }
3385
3386   /* try static mappings with port */
3387   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3388     {
3389       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3390                   clib_net_to_host_u16 (port));
3391       if (!clib_bihash_search_8_8
3392           (&sm->static_mapping_by_external, &kv, &value))
3393         {
3394           m = pool_elt_at_index (sm->static_mappings, value.value);
3395           if (!is_lb_static_mapping (m))
3396             {
3397               next_worker_index = m->workers[0];
3398               goto done;
3399             }
3400
3401           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3402             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3403
3404           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3405             next_worker_index =
3406               m->workers[hash & (_vec_len (m->workers) - 1)];
3407           else
3408             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3409           goto done;
3410         }
3411     }
3412
3413   /* worker by outside port */
3414   next_worker_index = sm->first_worker_index;
3415   next_worker_index +=
3416     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3417
3418 done:
3419   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3420                           clib_net_to_host_u32 (ip->src_address.as_u32),
3421                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3422   return next_worker_index;
3423 }
3424
3425 void
3426 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3427                 ip4_address_t * out_addr, u16 out_port,
3428                 ip4_address_t * eh_addr, u16 eh_port,
3429                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3430                 u32 fib_index, u16 flags, u32 thread_index)
3431 {
3432   snat_main_t *sm = &snat_main;
3433   snat_session_key_t key;
3434   snat_user_t *u;
3435   snat_session_t *s;
3436   clib_bihash_kv_8_8_t kv;
3437   f64 now = vlib_time_now (sm->vlib_main);
3438   nat_outside_fib_t *outside_fib;
3439   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3440   snat_main_per_thread_data_t *tsm;
3441   fib_prefix_t pfx = {
3442     .fp_proto = FIB_PROTOCOL_IP4,
3443     .fp_len = 32,
3444     .fp_addr = {
3445                 .ip4.as_u32 = eh_addr->as_u32,
3446                 },
3447   };
3448
3449   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3450
3451   key.addr.as_u32 = out_addr->as_u32;
3452   key.port = out_port;
3453   key.protocol = proto;
3454
3455   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3456     {
3457       if (nat_set_outside_address_and_port
3458           (sm->addresses, thread_index, &key))
3459         return;
3460     }
3461
3462   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3463   if (!u)
3464     return;
3465
3466   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3467   if (!s)
3468     return;
3469
3470   s->last_heard = now;
3471   s->flags = flags;
3472   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3473   s->ext_host_port = eh_port;
3474   user_session_increment (sm, u, snat_is_session_static (s));
3475   switch (vec_len (sm->outside_fibs))
3476     {
3477     case 0:
3478       key.fib_index = sm->outside_fib_index;
3479       break;
3480     case 1:
3481       key.fib_index = sm->outside_fibs[0].fib_index;
3482       break;
3483     default:
3484       /* *INDENT-OFF* */
3485       vec_foreach (outside_fib, sm->outside_fibs)
3486         {
3487           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3488           if (FIB_NODE_INDEX_INVALID != fei)
3489             {
3490               if (fib_entry_get_resolving_interface (fei) != ~0)
3491                 {
3492                   key.fib_index = outside_fib->fib_index;
3493                   break;
3494                 }
3495             }
3496         }
3497       /* *INDENT-ON* */
3498       break;
3499     }
3500   s->out2in = key;
3501   kv.key = key.as_u64;
3502   kv.value = s - tsm->sessions;
3503   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3504     nat_elog_warn ("out2in key add failed");
3505
3506   key.addr.as_u32 = in_addr->as_u32;
3507   key.port = in_port;
3508   key.fib_index = fib_index;
3509   s->in2out = key;
3510   kv.key = key.as_u64;
3511   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3512     nat_elog_warn ("in2out key add failed");
3513 }
3514
3515 void
3516 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3517                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3518                 u32 ti)
3519 {
3520   snat_main_t *sm = &snat_main;
3521   snat_session_key_t key;
3522   clib_bihash_kv_8_8_t kv, value;
3523   u32 thread_index;
3524   snat_session_t *s;
3525   snat_main_per_thread_data_t *tsm;
3526
3527   if (sm->num_workers > 1)
3528     thread_index =
3529       sm->first_worker_index +
3530       (sm->workers[(clib_net_to_host_u16 (out_port) -
3531                     1024) / sm->port_per_thread]);
3532   else
3533     thread_index = sm->num_workers;
3534   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3535
3536   key.addr.as_u32 = out_addr->as_u32;
3537   key.port = out_port;
3538   key.protocol = proto;
3539   key.fib_index = fib_index;
3540   kv.key = key.as_u64;
3541   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3542     return;
3543
3544   s = pool_elt_at_index (tsm->sessions, value.value);
3545   nat_free_session_data (sm, s, thread_index, 1);
3546   nat44_delete_session (sm, s, thread_index);
3547 }
3548
3549 void
3550 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3551                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3552                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3553 {
3554   snat_main_t *sm = &snat_main;
3555   snat_session_key_t key;
3556   clib_bihash_kv_8_8_t kv, value;
3557   snat_session_t *s;
3558   snat_main_per_thread_data_t *tsm;
3559
3560   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3561
3562   key.addr.as_u32 = out_addr->as_u32;
3563   key.port = out_port;
3564   key.protocol = proto;
3565   key.fib_index = fib_index;
3566   kv.key = key.as_u64;
3567   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3568     return;
3569
3570   s = pool_elt_at_index (tsm->sessions, value.value);
3571   s->total_pkts = total_pkts;
3572   s->total_bytes = total_bytes;
3573 }
3574
3575 void
3576 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3577                    ip4_address_t * out_addr, u16 out_port,
3578                    ip4_address_t * eh_addr, u16 eh_port,
3579                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3580                    u32 fib_index, u16 flags, u32 thread_index)
3581 {
3582   snat_main_t *sm = &snat_main;
3583   snat_session_key_t key;
3584   snat_user_t *u;
3585   snat_session_t *s;
3586   clib_bihash_kv_16_8_t kv;
3587   f64 now = vlib_time_now (sm->vlib_main);
3588   nat_outside_fib_t *outside_fib;
3589   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3590   snat_main_per_thread_data_t *tsm;
3591   fib_prefix_t pfx = {
3592     .fp_proto = FIB_PROTOCOL_IP4,
3593     .fp_len = 32,
3594     .fp_addr = {
3595                 .ip4.as_u32 = eh_addr->as_u32,
3596                 },
3597   };
3598
3599   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3600
3601   key.addr.as_u32 = out_addr->as_u32;
3602   key.port = out_port;
3603   key.protocol = proto;
3604
3605   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3606     {
3607       if (nat_set_outside_address_and_port
3608           (sm->addresses, thread_index, &key))
3609         return;
3610     }
3611
3612   key.addr.as_u32 = ehn_addr->as_u32;
3613   key.port = ehn_port;
3614   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3615     {
3616       if (nat_set_outside_address_and_port
3617           (sm->twice_nat_addresses, thread_index, &key))
3618         return;
3619     }
3620
3621   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3622   if (!u)
3623     return;
3624
3625   s = nat_ed_session_alloc (sm, u, thread_index, now);
3626   if (!s)
3627     return;
3628
3629   s->last_heard = now;
3630   s->flags = flags;
3631   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3632   s->ext_host_nat_port = s->ext_host_port = eh_port;
3633   if (is_twice_nat_session (s))
3634     {
3635       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3636       s->ext_host_nat_port = ehn_port;
3637     }
3638   user_session_increment (sm, u, snat_is_session_static (s));
3639   switch (vec_len (sm->outside_fibs))
3640     {
3641     case 0:
3642       key.fib_index = sm->outside_fib_index;
3643       break;
3644     case 1:
3645       key.fib_index = sm->outside_fibs[0].fib_index;
3646       break;
3647     default:
3648       /* *INDENT-OFF* */
3649       vec_foreach (outside_fib, sm->outside_fibs)
3650         {
3651           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3652           if (FIB_NODE_INDEX_INVALID != fei)
3653             {
3654               if (fib_entry_get_resolving_interface (fei) != ~0)
3655                 {
3656                   key.fib_index = outside_fib->fib_index;
3657                   break;
3658                 }
3659             }
3660         }
3661       /* *INDENT-ON* */
3662       break;
3663     }
3664   key.addr.as_u32 = out_addr->as_u32;
3665   key.port = out_port;
3666   s->out2in = key;
3667   kv.value = s - tsm->sessions;
3668
3669   key.addr.as_u32 = in_addr->as_u32;
3670   key.port = in_port;
3671   key.fib_index = fib_index;
3672   s->in2out = key;
3673
3674   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3675               snat_proto_to_ip_proto (proto), fib_index, in_port,
3676               s->ext_host_nat_port);
3677   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3678     nat_elog_warn ("in2out key add failed");
3679
3680   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3681               s->out2in.fib_index, out_port, eh_port);
3682   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3683     nat_elog_warn ("out2in key add failed");
3684 }
3685
3686 void
3687 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3688                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3689                    u32 fib_index, u32 ti)
3690 {
3691   snat_main_t *sm = &snat_main;
3692   nat_ed_ses_key_t key;
3693   clib_bihash_kv_16_8_t kv, value;
3694   u32 thread_index;
3695   snat_session_t *s;
3696   snat_main_per_thread_data_t *tsm;
3697
3698   if (sm->num_workers > 1)
3699     thread_index =
3700       sm->first_worker_index +
3701       (sm->workers[(clib_net_to_host_u16 (out_port) -
3702                     1024) / sm->port_per_thread]);
3703   else
3704     thread_index = sm->num_workers;
3705   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3706
3707   key.l_addr.as_u32 = out_addr->as_u32;
3708   key.l_port = out_port;
3709   key.r_addr.as_u32 = eh_addr->as_u32;
3710   key.r_port = eh_port;
3711   key.proto = proto;
3712   key.fib_index = fib_index;
3713   kv.key[0] = key.as_u64[0];
3714   kv.key[1] = key.as_u64[1];
3715   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3716     return;
3717
3718   s = pool_elt_at_index (tsm->sessions, value.value);
3719   nat_free_session_data (sm, s, thread_index, 1);
3720   nat44_delete_session (sm, s, thread_index);
3721 }
3722
3723 void
3724 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3725                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3726                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3727                    u32 thread_index)
3728 {
3729   snat_main_t *sm = &snat_main;
3730   nat_ed_ses_key_t key;
3731   clib_bihash_kv_16_8_t kv, value;
3732   snat_session_t *s;
3733   snat_main_per_thread_data_t *tsm;
3734
3735   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3736
3737   key.l_addr.as_u32 = out_addr->as_u32;
3738   key.l_port = out_port;
3739   key.r_addr.as_u32 = eh_addr->as_u32;
3740   key.r_port = eh_port;
3741   key.proto = proto;
3742   key.fib_index = fib_index;
3743   kv.key[0] = key.as_u64[0];
3744   kv.key[1] = key.as_u64[1];
3745   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3746     return;
3747
3748   s = pool_elt_at_index (tsm->sessions, value.value);
3749   s->total_pkts = total_pkts;
3750   s->total_bytes = total_bytes;
3751 }
3752
3753 static clib_error_t *
3754 snat_config (vlib_main_t * vm, unformat_input_t * input)
3755 {
3756   snat_main_t *sm = &snat_main;
3757   nat66_main_t *nm = &nat66_main;
3758   dslite_main_t *dm = &dslite_main;
3759   snat_main_per_thread_data_t *tsm;
3760
3761   u32 static_mapping_buckets = 1024;
3762   u32 static_mapping_memory_size = 64 << 20;
3763
3764   u32 nat64_bib_buckets = 1024;
3765   u32 nat64_bib_memory_size = 128 << 20;
3766
3767   u32 nat64_st_buckets = 2048;
3768   u32 nat64_st_memory_size = 256 << 20;
3769
3770   u32 user_buckets = 128;
3771   u32 user_memory_size = 64 << 20;
3772   u32 translation_buckets = 1024;
3773   u32 translation_memory_size = 128 << 20;
3774
3775   u32 max_translations_per_user = ~0;
3776
3777   u32 outside_vrf_id = 0;
3778   u32 outside_ip6_vrf_id = 0;
3779   u32 inside_vrf_id = 0;
3780   u8 static_mapping_only = 0;
3781   u8 static_mapping_connection_tracking = 0;
3782
3783   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3784   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3785
3786   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3787   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3788
3789   sm->deterministic = 0;
3790   sm->out2in_dpo = 0;
3791   sm->endpoint_dependent = 0;
3792
3793   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3794     {
3795       if (unformat
3796           (input, "translation hash buckets %d", &translation_buckets))
3797         ;
3798       else if (unformat (input, "udp timeout %d", &udp_timeout))
3799         ;
3800       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3801         ;
3802       else if (unformat (input, "tcp transitory timeout %d",
3803                          &tcp_transitory_timeout));
3804       else if (unformat (input, "tcp established timeout %d",
3805                          &tcp_established_timeout));
3806       else if (unformat (input, "translation hash memory %d",
3807                          &translation_memory_size));
3808       else if (unformat (input, "user hash buckets %d", &user_buckets))
3809         ;
3810       else if (unformat (input, "user hash memory %d", &user_memory_size))
3811         ;
3812       else if (unformat (input, "max translations per user %d",
3813                          &max_translations_per_user))
3814         ;
3815       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3816         ;
3817       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3818         ;
3819       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3820         ;
3821       else if (unformat (input, "static mapping only"))
3822         {
3823           static_mapping_only = 1;
3824           if (unformat (input, "connection tracking"))
3825             static_mapping_connection_tracking = 1;
3826         }
3827       else if (unformat (input, "deterministic"))
3828         sm->deterministic = 1;
3829       else if (unformat (input, "nat64 bib hash buckets %d",
3830                          &nat64_bib_buckets))
3831         ;
3832       else if (unformat (input, "nat64 bib hash memory %d",
3833                          &nat64_bib_memory_size))
3834         ;
3835       else
3836         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3837         ;
3838       else if (unformat (input, "nat64 st hash memory %d",
3839                          &nat64_st_memory_size))
3840         ;
3841       else if (unformat (input, "out2in dpo"))
3842         sm->out2in_dpo = 1;
3843       else if (unformat (input, "dslite ce"))
3844         dslite_set_ce (dm, 1);
3845       else if (unformat (input, "endpoint-dependent"))
3846         sm->endpoint_dependent = 1;
3847       else
3848         return clib_error_return (0, "unknown input '%U'",
3849                                   format_unformat_error, input);
3850     }
3851
3852   if (sm->deterministic && sm->endpoint_dependent)
3853     return clib_error_return (0,
3854                               "deterministic and endpoint-dependent modes are mutually exclusive");
3855
3856   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3857     return clib_error_return (0,
3858                               "static mapping only mode available only for simple nat");
3859
3860   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3861     return clib_error_return (0,
3862                               "out2in dpo mode available only for simple nat");
3863
3864   /* optionally configurable timeouts for testing purposes */
3865   sm->udp_timeout = udp_timeout;
3866   sm->icmp_timeout = icmp_timeout;
3867   sm->tcp_transitory_timeout = tcp_transitory_timeout;
3868   sm->tcp_established_timeout = tcp_established_timeout;
3869
3870   sm->user_buckets = user_buckets;
3871   sm->user_memory_size = user_memory_size;
3872
3873   sm->translation_buckets = translation_buckets;
3874   sm->translation_memory_size = translation_memory_size;
3875
3876   /* do not exceed load factor 10 */
3877   sm->max_translations = 10 * translation_buckets;
3878   sm->max_translations_per_user = max_translations_per_user == ~0 ?
3879     sm->max_translations : max_translations_per_user;
3880
3881   sm->outside_vrf_id = outside_vrf_id;
3882   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3883                                                              outside_vrf_id,
3884                                                              nat_fib_src_hi);
3885   nm->outside_vrf_id = outside_ip6_vrf_id;
3886   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3887                                                              outside_ip6_vrf_id,
3888                                                              nat_fib_src_hi);
3889   sm->inside_vrf_id = inside_vrf_id;
3890   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3891                                                             inside_vrf_id,
3892                                                             nat_fib_src_hi);
3893   sm->static_mapping_only = static_mapping_only;
3894   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3895
3896   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3897                   nat64_st_memory_size);
3898
3899   if (sm->deterministic)
3900     {
3901       sm->in2out_node_index = snat_det_in2out_node.index;
3902       sm->in2out_output_node_index = ~0;
3903       sm->out2in_node_index = snat_det_out2in_node.index;
3904       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3905       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3906     }
3907   else
3908     {
3909       if (sm->endpoint_dependent)
3910         {
3911           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
3912           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3913
3914           sm->handoff_out2in_index = nat_pre_out2in_node.index;
3915           sm->handoff_in2out_index = nat_pre_in2out_node.index;
3916           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
3917
3918           sm->in2out_node_index = nat44_ed_in2out_node.index;
3919           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3920           sm->out2in_node_index = nat44_ed_out2in_node.index;
3921
3922           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3923           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3924           nat_affinity_init (vm);
3925           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
3926                        nat_ha_sref_ed_cb);
3927         }
3928       else
3929         {
3930           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3931           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3932
3933           sm->handoff_out2in_index = snat_in2out_node.index;
3934           sm->handoff_in2out_index = snat_out2in_node.index;
3935           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
3936
3937           sm->in2out_node_index = snat_in2out_node.index;
3938           sm->in2out_output_node_index = snat_in2out_output_node.index;
3939           sm->out2in_node_index = snat_out2in_node.index;
3940           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3941           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3942           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
3943         }
3944       if (!static_mapping_only ||
3945           (static_mapping_only && static_mapping_connection_tracking))
3946         {
3947           /* *INDENT-OFF* */
3948           vec_foreach (tsm, sm->per_thread_data)
3949             {
3950               if (sm->endpoint_dependent)
3951                 {
3952                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3953                                          translation_buckets,
3954                                          translation_memory_size);
3955                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3956                                                       format_ed_session_kvp);
3957
3958                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3959                                          translation_buckets,
3960                                          translation_memory_size);
3961                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3962                                                       format_ed_session_kvp);
3963                 }
3964               else
3965                 {
3966                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3967                                         translation_buckets,
3968                                         translation_memory_size);
3969                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3970                                                      format_session_kvp);
3971
3972                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3973                                         translation_buckets,
3974                                         translation_memory_size);
3975                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3976                                                      format_session_kvp);
3977                 }
3978
3979               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3980                                     user_memory_size);
3981               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3982                                                  format_user_kvp);
3983             }
3984           /* *INDENT-ON* */
3985
3986         }
3987       else
3988         {
3989           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3990           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3991         }
3992       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3993                             "static_mapping_by_local", static_mapping_buckets,
3994                             static_mapping_memory_size);
3995       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3996                                          format_static_mapping_kvp);
3997
3998       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3999                             "static_mapping_by_external",
4000                             static_mapping_buckets,
4001                             static_mapping_memory_size);
4002       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4003                                          format_static_mapping_kvp);
4004     }
4005
4006   return 0;
4007 }
4008
4009 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4010
4011 static void
4012 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4013                                  uword opaque,
4014                                  u32 sw_if_index,
4015                                  ip4_address_t * address,
4016                                  u32 address_length,
4017                                  u32 if_address_index, u32 is_delete)
4018 {
4019   snat_main_t *sm = &snat_main;
4020   snat_static_map_resolve_t *rp;
4021   snat_static_mapping_t *m;
4022   snat_session_key_t m_key;
4023   clib_bihash_kv_8_8_t kv, value;
4024   int i, rv;
4025   ip4_address_t l_addr;
4026
4027   for (i = 0; i < vec_len (sm->to_resolve); i++)
4028     {
4029       rp = sm->to_resolve + i;
4030       if (rp->addr_only == 0)
4031         continue;
4032       if (rp->sw_if_index == sw_if_index)
4033         goto match;
4034     }
4035
4036   return;
4037
4038 match:
4039   m_key.addr.as_u32 = address->as_u32;
4040   m_key.port = rp->addr_only ? 0 : rp->e_port;
4041   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4042   m_key.fib_index = sm->outside_fib_index;
4043   kv.key = m_key.as_u64;
4044   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4045     m = 0;
4046   else
4047     m = pool_elt_at_index (sm->static_mappings, value.value);
4048
4049   if (!is_delete)
4050     {
4051       /* Don't trip over lease renewal, static config */
4052       if (m)
4053         return;
4054     }
4055   else
4056     {
4057       if (!m)
4058         return;
4059     }
4060
4061   /* Indetity mapping? */
4062   if (rp->l_addr.as_u32 == 0)
4063     l_addr.as_u32 = address[0].as_u32;
4064   else
4065     l_addr.as_u32 = rp->l_addr.as_u32;
4066   /* Add the static mapping */
4067   rv = snat_add_static_mapping (l_addr,
4068                                 address[0],
4069                                 rp->l_port,
4070                                 rp->e_port,
4071                                 rp->vrf_id,
4072                                 rp->addr_only, ~0 /* sw_if_index */ ,
4073                                 rp->proto, !is_delete, rp->twice_nat,
4074                                 rp->out2in_only, rp->tag, rp->identity_nat);
4075   if (rv)
4076     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4077 }
4078
4079 static void
4080 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4081                                        uword opaque,
4082                                        u32 sw_if_index,
4083                                        ip4_address_t * address,
4084                                        u32 address_length,
4085                                        u32 if_address_index, u32 is_delete)
4086 {
4087   snat_main_t *sm = &snat_main;
4088   snat_static_map_resolve_t *rp;
4089   ip4_address_t l_addr;
4090   int i, j;
4091   int rv;
4092   u8 twice_nat = 0;
4093   snat_address_t *addresses = sm->addresses;
4094
4095   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4096     {
4097       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4098         goto match;
4099     }
4100
4101   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4102     {
4103       twice_nat = 1;
4104       addresses = sm->twice_nat_addresses;
4105       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4106         goto match;
4107     }
4108
4109   return;
4110
4111 match:
4112   if (!is_delete)
4113     {
4114       /* Don't trip over lease renewal, static config */
4115       for (j = 0; j < vec_len (addresses); j++)
4116         if (addresses[j].addr.as_u32 == address->as_u32)
4117           return;
4118
4119       (void) snat_add_address (sm, address, ~0, twice_nat);
4120       /* Scan static map resolution vector */
4121       for (j = 0; j < vec_len (sm->to_resolve); j++)
4122         {
4123           rp = sm->to_resolve + j;
4124           if (rp->addr_only)
4125             continue;
4126           /* On this interface? */
4127           if (rp->sw_if_index == sw_if_index)
4128             {
4129               /* Indetity mapping? */
4130               if (rp->l_addr.as_u32 == 0)
4131                 l_addr.as_u32 = address[0].as_u32;
4132               else
4133                 l_addr.as_u32 = rp->l_addr.as_u32;
4134               /* Add the static mapping */
4135               rv = snat_add_static_mapping (l_addr,
4136                                             address[0],
4137                                             rp->l_port,
4138                                             rp->e_port,
4139                                             rp->vrf_id,
4140                                             rp->addr_only,
4141                                             ~0 /* sw_if_index */ ,
4142                                             rp->proto,
4143                                             rp->is_add, rp->twice_nat,
4144                                             rp->out2in_only, rp->tag,
4145                                             rp->identity_nat);
4146               if (rv)
4147                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4148                                     "i4", rv);
4149             }
4150         }
4151       return;
4152     }
4153   else
4154     {
4155       (void) snat_del_address (sm, address[0], 1, twice_nat);
4156       return;
4157     }
4158 }
4159
4160
4161 int
4162 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4163                             u8 twice_nat)
4164 {
4165   ip4_main_t *ip4_main = sm->ip4_main;
4166   ip4_address_t *first_int_addr;
4167   snat_static_map_resolve_t *rp;
4168   u32 *indices_to_delete = 0;
4169   int i, j;
4170   u32 *auto_add_sw_if_indices =
4171     twice_nat ? sm->
4172     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4173
4174   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4175     );
4176
4177   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4178     {
4179       if (auto_add_sw_if_indices[i] == sw_if_index)
4180         {
4181           if (is_del)
4182             {
4183               /* if have address remove it */
4184               if (first_int_addr)
4185                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4186               else
4187                 {
4188                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4189                     {
4190                       rp = sm->to_resolve + j;
4191                       if (rp->sw_if_index == sw_if_index)
4192                         vec_add1 (indices_to_delete, j);
4193                     }
4194                   if (vec_len (indices_to_delete))
4195                     {
4196                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4197                         vec_del1 (sm->to_resolve, j);
4198                       vec_free (indices_to_delete);
4199                     }
4200                 }
4201               if (twice_nat)
4202                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4203               else
4204                 vec_del1 (sm->auto_add_sw_if_indices, i);
4205             }
4206           else
4207             return VNET_API_ERROR_VALUE_EXIST;
4208
4209           return 0;
4210         }
4211     }
4212
4213   if (is_del)
4214     return VNET_API_ERROR_NO_SUCH_ENTRY;
4215
4216   /* add to the auto-address list */
4217   if (twice_nat)
4218     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4219   else
4220     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4221
4222   /* If the address is already bound - or static - add it now */
4223   if (first_int_addr)
4224     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4225
4226   return 0;
4227 }
4228
4229 int
4230 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4231                    snat_protocol_t proto, u32 vrf_id, int is_in)
4232 {
4233   snat_main_per_thread_data_t *tsm;
4234   clib_bihash_kv_8_8_t kv, value;
4235   ip4_header_t ip;
4236   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4237   snat_session_key_t key;
4238   snat_session_t *s;
4239   clib_bihash_8_8_t *t;
4240
4241   if (sm->endpoint_dependent)
4242     return VNET_API_ERROR_UNSUPPORTED;
4243
4244   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4245   if (sm->num_workers > 1)
4246     tsm =
4247       vec_elt_at_index (sm->per_thread_data,
4248                         sm->worker_in2out_cb (&ip, fib_index, 0));
4249   else
4250     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4251
4252   key.addr.as_u32 = addr->as_u32;
4253   key.port = clib_host_to_net_u16 (port);
4254   key.protocol = proto;
4255   key.fib_index = fib_index;
4256   kv.key = key.as_u64;
4257   t = is_in ? &tsm->in2out : &tsm->out2in;
4258   if (!clib_bihash_search_8_8 (t, &kv, &value))
4259     {
4260       if (pool_is_free_index (tsm->sessions, value.value))
4261         return VNET_API_ERROR_UNSPECIFIED;
4262
4263       s = pool_elt_at_index (tsm->sessions, value.value);
4264       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4265       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4266       return 0;
4267     }
4268
4269   return VNET_API_ERROR_NO_SUCH_ENTRY;
4270 }
4271
4272 int
4273 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4274                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4275                       u32 vrf_id, int is_in)
4276 {
4277   ip4_header_t ip;
4278   clib_bihash_16_8_t *t;
4279   nat_ed_ses_key_t key;
4280   clib_bihash_kv_16_8_t kv, value;
4281   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4282   snat_session_t *s;
4283   snat_main_per_thread_data_t *tsm;
4284
4285   if (!sm->endpoint_dependent)
4286     return VNET_API_ERROR_FEATURE_DISABLED;
4287
4288   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4289   if (sm->num_workers > 1)
4290     tsm =
4291       vec_elt_at_index (sm->per_thread_data,
4292                         sm->worker_in2out_cb (&ip, fib_index, 0));
4293   else
4294     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4295
4296   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4297   key.l_addr.as_u32 = addr->as_u32;
4298   key.r_addr.as_u32 = eh_addr->as_u32;
4299   key.l_port = clib_host_to_net_u16 (port);
4300   key.r_port = clib_host_to_net_u16 (eh_port);
4301   key.proto = proto;
4302   key.fib_index = fib_index;
4303   kv.key[0] = key.as_u64[0];
4304   kv.key[1] = key.as_u64[1];
4305   if (clib_bihash_search_16_8 (t, &kv, &value))
4306     return VNET_API_ERROR_NO_SUCH_ENTRY;
4307
4308   if (pool_is_free_index (tsm->sessions, value.value))
4309     return VNET_API_ERROR_UNSPECIFIED;
4310   s = pool_elt_at_index (tsm->sessions, value.value);
4311   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4312   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4313   return 0;
4314 }
4315
4316 void
4317 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4318 {
4319   snat_main_t *sm = &snat_main;
4320
4321   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4322   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4323   sm->psid = psid;
4324   sm->psid_offset = psid_offset;
4325   sm->psid_length = psid_length;
4326 }
4327
4328 void
4329 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4330 {
4331   snat_main_t *sm = &snat_main;
4332
4333   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4334   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4335   sm->start_port = start_port;
4336   sm->end_port = end_port;
4337 }
4338
4339 void
4340 nat_set_alloc_addr_and_port_default (void)
4341 {
4342   snat_main_t *sm = &snat_main;
4343
4344   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4345   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4346 }
4347
4348 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4349                                  vlib_node_runtime_t * node,
4350                                  vlib_frame_t * frame)
4351 {
4352   return 0;
4353 }
4354
4355 /* *INDENT-OFF* */
4356 VLIB_REGISTER_NODE (nat_default_node) = {
4357   .name = "nat-default",
4358   .vector_size = sizeof (u32),
4359   .format_trace = 0,
4360   .type = VLIB_NODE_TYPE_INTERNAL,
4361   .n_errors = 0,
4362   .n_next_nodes = NAT_N_NEXT,
4363   .next_nodes = {
4364     [NAT_NEXT_DROP] = "error-drop",
4365     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4366     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4367     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4368     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4369     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4370     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4371     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4372     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4373   },
4374 };
4375 /* *INDENT-ON* */
4376
4377 /*
4378  * fd.io coding-style-patch-verification: ON
4379  *
4380  * Local Variables:
4381  * eval: (c-set-style "gnu")
4382  * End:
4383  */