src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_inlines.h>
  30 #include <nat/nat_affinity.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35 #include <vnet/ip/reass/ip4_sv_reass.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 fib_source_t nat_fib_src_hi;
  42 fib_source_t nat_fib_src_low;
  43
  44 /* *INDENT-OFF* */
  45 /* Hook up input features */
  46 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat-pre-in2out",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-sv-reassembly-feature"),
  51 };
  52 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat-pre-out2in",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  56                                "ip4-dhcp-client-detect",
  57                                "ip4-sv-reassembly-feature"),
  58 };
  59 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-in2out-worker-handoff",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-out2in-worker-handoff",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-in2out",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  74 };
  75 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-out2in",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  79                                "ip4-dhcp-client-detect"),
  80 };
  81 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-classify",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  85 };
  86 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-det-in2out",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-det-out2in",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  95                                "ip4-dhcp-client-detect"),
  96 };
  97 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-det-classify",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-ed-in2out",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 106 };
 107 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-ed-out2in",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 111                                "ip4-dhcp-client-detect"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-ed-classify",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-handoff-classify",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 122 };
 123 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-in2out-fast",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 127 };
 128 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 129   .arc_name = "ip4-unicast",
 130   .node_name = "nat44-out2in-fast",
 131   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 132                                "ip4-dhcp-client-detect"),
 133 };
 134 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 135   .arc_name = "ip4-unicast",
 136   .node_name = "nat44-hairpin-dst",
 137   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 140   .arc_name = "ip4-unicast",
 141   .node_name = "nat44-ed-hairpin-dst",
 142   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 143 };
 144
 145 /* Hook up output features */
 146 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-in2out-output",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-in2out-output-worker-handoff",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 155 };
 156 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 157   .arc_name = "ip4-output",
 158   .node_name = "nat44-hairpin-src",
 159   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 162   .arc_name = "ip4-output",
 163   .node_name = "nat44-ed-in2out-output",
 164   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 165 };
 166 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 167   .arc_name = "ip4-output",
 168   .node_name = "nat44-ed-hairpin-src",
 169   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 170 };
 171
 172 /* Hook up ip4-local features */
 173 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 174 {
 175   .arc_name = "ip4-local",
 176   .node_name = "nat44-hairpinning",
 177   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 178 };
 179 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 180 {
 181   .arc_name = "ip4-local",
 182   .node_name = "nat44-ed-hairpinning",
 183   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 184 };
 185
 186
 187 VLIB_PLUGIN_REGISTER () = {
 188     .version = VPP_BUILD_VER,
 189     .description = "Network Address Translation (NAT)",
 190 };
 191 /* *INDENT-ON* */
 192
 193 void
 194 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 195                        u8 is_ha)
 196 {
 197   snat_session_key_t key;
 198   clib_bihash_kv_8_8_t kv;
 199   nat_ed_ses_key_t ed_key;
 200   clib_bihash_kv_16_8_t ed_kv;
 201   snat_main_per_thread_data_t *tsm =
 202     vec_elt_at_index (sm->per_thread_data, thread_index);
 203
 204   if (is_fwd_bypass_session (s))
 205     {
 206       if (snat_is_unk_proto_session (s))
 207         {
 208           ed_key.proto = s->in2out.port;
 209           ed_key.r_port = 0;
 210           ed_key.l_port = 0;
 211         }
 212       else
 213         {
 214           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 215           ed_key.l_port = s->in2out.port;
 216           ed_key.r_port = s->ext_host_port;
 217         }
 218       ed_key.l_addr = s->in2out.addr;
 219       ed_key.r_addr = s->ext_host_addr;
 220       ed_key.fib_index = 0;
 221       ed_kv.key[0] = ed_key.as_u64[0];
 222       ed_kv.key[1] = ed_key.as_u64[1];
 223       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 224         nat_elog_warn ("in2out_ed key del failed");
 225       return;
 226     }
 227
 228   /* session lookup tables */
 229   if (is_ed_session (s))
 230     {
 231       if (is_affinity_sessions (s))
 232         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 233                              s->in2out.protocol, s->out2in.port);
 234       ed_key.l_addr = s->out2in.addr;
 235       ed_key.r_addr = s->ext_host_addr;
 236       ed_key.fib_index = s->out2in.fib_index;
 237       if (snat_is_unk_proto_session (s))
 238         {
 239           ed_key.proto = s->in2out.port;
 240           ed_key.r_port = 0;
 241           ed_key.l_port = 0;
 242         }
 243       else
 244         {
 245           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 246           ed_key.l_port = s->out2in.port;
 247           ed_key.r_port = s->ext_host_port;
 248         }
 249       ed_kv.key[0] = ed_key.as_u64[0];
 250       ed_kv.key[1] = ed_key.as_u64[1];
 251       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 252         nat_elog_warn ("out2in_ed key del failed");
 253       ed_key.l_addr = s->in2out.addr;
 254       ed_key.fib_index = s->in2out.fib_index;
 255       if (!snat_is_unk_proto_session (s))
 256         ed_key.l_port = s->in2out.port;
 257       if (is_twice_nat_session (s))
 258         {
 259           ed_key.r_addr = s->ext_host_nat_addr;
 260           ed_key.r_port = s->ext_host_nat_port;
 261         }
 262       ed_kv.key[0] = ed_key.as_u64[0];
 263       ed_kv.key[1] = ed_key.as_u64[1];
 264       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 265         nat_elog_warn ("in2out_ed key del failed");
 266
 267       if (!is_ha)
 268         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 269                                &s->in2out.addr, s->in2out.port,
 270                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 271                                &s->out2in.addr, s->out2in.port,
 272                                &s->ext_host_addr, s->ext_host_port,
 273                                s->in2out.protocol, is_twice_nat_session (s));
 274     }
 275   else
 276     {
 277       kv.key = s->in2out.as_u64;
 278       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 279         nat_elog_warn ("in2out key del failed");
 280       kv.key = s->out2in.as_u64;
 281       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 282         nat_elog_warn ("out2in key del failed");
 283
 284       if (!is_ha)
 285         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 286                                  &s->in2out.addr, s->in2out.port,
 287                                  &s->out2in.addr, s->out2in.port,
 288                                  s->in2out.protocol);
 289     }
 290
 291   if (snat_is_unk_proto_session (s))
 292     return;
 293
 294   if (!is_ha)
 295     {
 296       /* log NAT event */
 297       snat_ipfix_logging_nat44_ses_delete (thread_index,
 298                                            s->in2out.addr.as_u32,
 299                                            s->out2in.addr.as_u32,
 300                                            s->in2out.protocol,
 301                                            s->in2out.port,
 302                                            s->out2in.port,
 303                                            s->in2out.fib_index);
 304
 305       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 306                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 307                    thread_index);
 308     }
 309
 310   /* Twice NAT address and port for external host */
 311   if (is_twice_nat_session (s))
 312     {
 313       key.protocol = s->in2out.protocol;
 314       key.port = s->ext_host_nat_port;
 315       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 316       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 317                                           thread_index, &key);
 318     }
 319
 320   if (snat_is_session_static (s))
 321     return;
 322
 323   snat_free_outside_address_and_port (sm->addresses, thread_index,
 324                                       &s->out2in);
 325 }
 326
 327 snat_user_t *
 328 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 329                         u32 thread_index)
 330 {
 331   snat_user_t *u = 0;
 332   snat_user_key_t user_key;
 333   clib_bihash_kv_8_8_t kv, value;
 334   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 335   dlist_elt_t *per_user_list_head_elt;
 336
 337   user_key.addr.as_u32 = addr->as_u32;
 338   user_key.fib_index = fib_index;
 339   kv.key = user_key.as_u64;
 340
 341   /* Ever heard of the "user" = src ip4 address before? */
 342   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 343     {
 344       /* no, make a new one */
 345       pool_get (tsm->users, u);
 346       clib_memset (u, 0, sizeof (*u));
 347       u->addr.as_u32 = addr->as_u32;
 348       u->fib_index = fib_index;
 349
 350       pool_get (tsm->list_pool, per_user_list_head_elt);
 351
 352       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 353         tsm->list_pool;
 354
 355       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 356
 357       kv.value = u - tsm->users;
 358
 359       /* add user */
 360       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 361         nat_elog_warn ("user_hash keay add failed");
 362
 363       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 364                                pool_elts (tsm->users));
 365     }
 366   else
 367     {
 368       u = pool_elt_at_index (tsm->users, value.value);
 369     }
 370
 371   return u;
 372 }
 373
 374 snat_session_t *
 375 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 376                               u32 thread_index, f64 now)
 377 {
 378   snat_session_t *s;
 379   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 380   u32 oldest_per_user_translation_list_index, session_index;
 381   dlist_elt_t *oldest_per_user_translation_list_elt;
 382   dlist_elt_t *per_user_translation_list_elt;
 383
 384   /* Over quota? Recycle the least recently used translation */
 385   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 386     {
 387       oldest_per_user_translation_list_index =
 388         clib_dlist_remove_head (tsm->list_pool,
 389                                 u->sessions_per_user_list_head_index);
 390
 391       ASSERT (oldest_per_user_translation_list_index != ~0);
 392
 393       /* Add it back to the end of the LRU list */
 394       clib_dlist_addtail (tsm->list_pool,
 395                           u->sessions_per_user_list_head_index,
 396                           oldest_per_user_translation_list_index);
 397       /* Get the list element */
 398       oldest_per_user_translation_list_elt =
 399         pool_elt_at_index (tsm->list_pool,
 400                            oldest_per_user_translation_list_index);
 401
 402       /* Get the session index from the list element */
 403       session_index = oldest_per_user_translation_list_elt->value;
 404
 405       /* Get the session */
 406       s = pool_elt_at_index (tsm->sessions, session_index);
 407       nat_free_session_data (sm, s, thread_index, 0);
 408       if (snat_is_session_static (s))
 409         u->nstaticsessions--;
 410       else
 411         u->nsessions--;
 412       s->flags = 0;
 413       s->total_bytes = 0;
 414       s->total_pkts = 0;
 415       s->state = 0;
 416       s->ext_host_addr.as_u32 = 0;
 417       s->ext_host_port = 0;
 418       s->ext_host_nat_addr.as_u32 = 0;
 419       s->ext_host_nat_port = 0;
 420     }
 421   else
 422     {
 423       pool_get (tsm->sessions, s);
 424       clib_memset (s, 0, sizeof (*s));
 425
 426       /* Create list elts */
 427       pool_get (tsm->list_pool, per_user_translation_list_elt);
 428       clib_dlist_init (tsm->list_pool,
 429                        per_user_translation_list_elt - tsm->list_pool);
 430
 431       per_user_translation_list_elt->value = s - tsm->sessions;
 432       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 433       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 434
 435       clib_dlist_addtail (tsm->list_pool,
 436                           s->per_user_list_head_index,
 437                           per_user_translation_list_elt - tsm->list_pool);
 438
 439       s->user_index = u - tsm->users;
 440       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 441                                pool_elts (tsm->sessions));
 442     }
 443
 444   s->ha_last_refreshed = now;
 445
 446   return s;
 447 }
 448
 449 snat_session_t *
 450 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 451                       f64 now)
 452 {
 453   snat_session_t *s;
 454   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 455   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 456   u32 oldest_index;
 457   u64 sess_timeout_time;
 458
 459   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 460     goto alloc_new;
 461
 462   oldest_index =
 463     clib_dlist_remove_head (tsm->list_pool,
 464                             u->sessions_per_user_list_head_index);
 465   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 466   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 467
 468   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 469   if (now >= sess_timeout_time)
 470     {
 471       clib_dlist_addtail (tsm->list_pool,
 472                           u->sessions_per_user_list_head_index, oldest_index);
 473       nat_free_session_data (sm, s, thread_index, 0);
 474       if (snat_is_session_static (s))
 475         u->nstaticsessions--;
 476       else
 477         u->nsessions--;
 478       s->flags = 0;
 479       s->total_bytes = 0;
 480       s->total_pkts = 0;
 481       s->state = 0;
 482       s->ext_host_addr.as_u32 = 0;
 483       s->ext_host_port = 0;
 484       s->ext_host_nat_addr.as_u32 = 0;
 485       s->ext_host_nat_port = 0;
 486     }
 487   else
 488     {
 489       clib_dlist_addhead (tsm->list_pool,
 490                           u->sessions_per_user_list_head_index, oldest_index);
 491       if ((u->nsessions + u->nstaticsessions) >=
 492           sm->max_translations_per_user)
 493         {
 494           nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 495                          clib_net_to_host_u32 (u->addr.as_u32));
 496           snat_ipfix_logging_max_entries_per_user
 497             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 498           return 0;
 499         }
 500       else
 501         {
 502         alloc_new:
 503           pool_get (tsm->sessions, s);
 504           clib_memset (s, 0, sizeof (*s));
 505
 506           /* Create list elts */
 507           pool_get (tsm->list_pool, per_user_translation_list_elt);
 508           clib_dlist_init (tsm->list_pool,
 509                            per_user_translation_list_elt - tsm->list_pool);
 510
 511           per_user_translation_list_elt->value = s - tsm->sessions;
 512           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 513           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 514
 515           clib_dlist_addtail (tsm->list_pool,
 516                               s->per_user_list_head_index,
 517                               per_user_translation_list_elt - tsm->list_pool);
 518         }
 519
 520       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 521                                pool_elts (tsm->sessions));
 522     }
 523
 524   s->ha_last_refreshed = now;
 525
 526   return s;
 527 }
 528
 529 void
 530 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 531                           int is_add)
 532 {
 533   fib_prefix_t prefix = {
 534     .fp_len = p_len,
 535     .fp_proto = FIB_PROTOCOL_IP4,
 536     .fp_addr = {
 537                 .ip4.as_u32 = addr->as_u32,
 538                 },
 539   };
 540   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 541
 542   if (is_add)
 543     fib_table_entry_update_one_path (fib_index,
 544                                      &prefix,
 545                                      nat_fib_src_low,
 546                                      (FIB_ENTRY_FLAG_CONNECTED |
 547                                       FIB_ENTRY_FLAG_LOCAL |
 548                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 549                                      DPO_PROTO_IP4,
 550                                      NULL,
 551                                      sw_if_index,
 552                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 553   else
 554     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 555 }
 556
 557 int
 558 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 559                   u8 twice_nat)
 560 {
 561   snat_address_t *ap;
 562   snat_interface_t *i;
 563   vlib_thread_main_t *tm = vlib_get_thread_main ();
 564
 565   if (twice_nat && !sm->endpoint_dependent)
 566     return VNET_API_ERROR_FEATURE_DISABLED;
 567
 568   /* Check if address already exists */
 569   /* *INDENT-OFF* */
 570   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 571     {
 572       if (ap->addr.as_u32 == addr->as_u32)
 573         return VNET_API_ERROR_VALUE_EXIST;
 574     }
 575   /* *INDENT-ON* */
 576
 577   if (twice_nat)
 578     vec_add2 (sm->twice_nat_addresses, ap, 1);
 579   else
 580     vec_add2 (sm->addresses, ap, 1);
 581
 582   ap->addr = *addr;
 583   if (vrf_id != ~0)
 584     ap->fib_index =
 585       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 586                                          nat_fib_src_low);
 587   else
 588     ap->fib_index = ~0;
 589 #define _(N, i, n, s) \
 590   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 591   ap->busy_##n##_ports = 0; \
 592   ap->busy_##n##_ports_per_thread = 0;\
 593   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 594   foreach_snat_protocol
 595 #undef _
 596     if (twice_nat)
 597     return 0;
 598
 599   /* Add external address to FIB */
 600   /* *INDENT-OFF* */
 601   pool_foreach (i, sm->interfaces,
 602   ({
 603     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 604       continue;
 605
 606     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 607     break;
 608   }));
 609   pool_foreach (i, sm->output_feature_interfaces,
 610   ({
 611     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 612       continue;
 613
 614     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 615     break;
 616   }));
 617   /* *INDENT-ON* */
 618
 619   return 0;
 620 }
 621
 622 static int
 623 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 624 {
 625   snat_static_mapping_t *m;
 626   /* *INDENT-OFF* */
 627   pool_foreach (m, sm->static_mappings,
 628   ({
 629       if (is_addr_only_static_mapping (m) ||
 630           is_out2in_only_static_mapping (m) ||
 631           is_identity_static_mapping (m))
 632         continue;
 633       if (m->external_addr.as_u32 == addr.as_u32)
 634         return 1;
 635   }));
 636   /* *INDENT-ON* */
 637
 638   return 0;
 639 }
 640
 641 void
 642 increment_v4_address (ip4_address_t * a)
 643 {
 644   u32 v;
 645
 646   v = clib_net_to_host_u32 (a->as_u32) + 1;
 647   a->as_u32 = clib_host_to_net_u32 (v);
 648 }
 649
 650 static void
 651 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 652                                        ip4_address_t l_addr,
 653                                        u16 l_port,
 654                                        u32 sw_if_index,
 655                                        u16 e_port,
 656                                        u32 vrf_id,
 657                                        snat_protocol_t proto,
 658                                        int addr_only, int is_add, u8 * tag,
 659                                        int twice_nat, int out2in_only,
 660                                        int identity_nat)
 661 {
 662   snat_static_map_resolve_t *rp;
 663
 664   vec_add2 (sm->to_resolve, rp, 1);
 665   rp->l_addr.as_u32 = l_addr.as_u32;
 666   rp->l_port = l_port;
 667   rp->sw_if_index = sw_if_index;
 668   rp->e_port = e_port;
 669   rp->vrf_id = vrf_id;
 670   rp->proto = proto;
 671   rp->addr_only = addr_only;
 672   rp->is_add = is_add;
 673   rp->twice_nat = twice_nat;
 674   rp->out2in_only = out2in_only;
 675   rp->identity_nat = identity_nat;
 676   rp->tag = vec_dup (tag);
 677 }
 678
 679 static u32
 680 get_thread_idx_by_port (u16 e_port)
 681 {
 682   snat_main_t *sm = &snat_main;
 683   u32 thread_idx = sm->num_workers;
 684   if (sm->num_workers > 1)
 685     {
 686       thread_idx =
 687         sm->first_worker_index +
 688         sm->workers[(e_port - 1024) / sm->port_per_thread];
 689     }
 690   return thread_idx;
 691 }
 692
 693 int
 694 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 695                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 696                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 697                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 698                          u8 identity_nat)
 699 {
 700   snat_main_t *sm = &snat_main;
 701   snat_static_mapping_t *m;
 702   snat_session_key_t m_key;
 703   clib_bihash_kv_8_8_t kv, value;
 704   snat_address_t *a = 0;
 705   u32 fib_index = ~0;
 706   snat_interface_t *interface;
 707   int i;
 708   snat_main_per_thread_data_t *tsm;
 709   snat_user_key_t u_key;
 710   snat_user_t *u;
 711   dlist_elt_t *head, *elt;
 712   u32 elt_index, head_index;
 713   u32 ses_index;
 714   u64 user_index;
 715   snat_session_t *s;
 716   snat_static_map_resolve_t *rp, *rp_match = 0;
 717   nat44_lb_addr_port_t *local;
 718   u32 find = ~0;
 719
 720   if (!sm->endpoint_dependent)
 721     {
 722       if (twice_nat || out2in_only)
 723         return VNET_API_ERROR_FEATURE_DISABLED;
 724     }
 725
 726   /* If the external address is a specific interface address */
 727   if (sw_if_index != ~0)
 728     {
 729       ip4_address_t *first_int_addr;
 730
 731       for (i = 0; i < vec_len (sm->to_resolve); i++)
 732         {
 733           rp = sm->to_resolve + i;
 734           if (rp->sw_if_index != sw_if_index ||
 735               rp->l_addr.as_u32 != l_addr.as_u32 ||
 736               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 737             continue;
 738
 739           if (!addr_only)
 740             {
 741               if ((rp->l_port != l_port && rp->e_port != e_port)
 742                   || rp->proto != proto)
 743                 continue;
 744             }
 745
 746           rp_match = rp;
 747           break;
 748         }
 749
 750       /* Might be already set... */
 751       first_int_addr = ip4_interface_first_address
 752         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 753
 754       if (is_add)
 755         {
 756           if (rp_match)
 757             return VNET_API_ERROR_VALUE_EXIST;
 758
 759           snat_add_static_mapping_when_resolved
 760             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 761              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 762
 763           /* DHCP resolution required? */
 764           if (first_int_addr == 0)
 765             {
 766               return 0;
 767             }
 768           else
 769             {
 770               e_addr.as_u32 = first_int_addr->as_u32;
 771               /* Identity mapping? */
 772               if (l_addr.as_u32 == 0)
 773                 l_addr.as_u32 = e_addr.as_u32;
 774             }
 775         }
 776       else
 777         {
 778           if (!rp_match)
 779             return VNET_API_ERROR_NO_SUCH_ENTRY;
 780
 781           vec_del1 (sm->to_resolve, i);
 782
 783           if (first_int_addr)
 784             {
 785               e_addr.as_u32 = first_int_addr->as_u32;
 786               /* Identity mapping? */
 787               if (l_addr.as_u32 == 0)
 788                 l_addr.as_u32 = e_addr.as_u32;
 789             }
 790           else
 791             return 0;
 792         }
 793     }
 794
 795   m_key.addr = e_addr;
 796   m_key.port = addr_only ? 0 : e_port;
 797   m_key.protocol = addr_only ? 0 : proto;
 798   m_key.fib_index = 0;
 799   kv.key = m_key.as_u64;
 800   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 801     m = 0;
 802   else
 803     m = pool_elt_at_index (sm->static_mappings, value.value);
 804
 805   if (is_add)
 806     {
 807       if (m)
 808         {
 809           if (is_identity_static_mapping (m))
 810             {
 811               /* *INDENT-OFF* */
 812               pool_foreach (local, m->locals,
 813               ({
 814                 if (local->vrf_id == vrf_id)
 815                   return VNET_API_ERROR_VALUE_EXIST;
 816               }));
 817               /* *INDENT-ON* */
 818               pool_get (m->locals, local);
 819               local->vrf_id = vrf_id;
 820               local->fib_index =
 821                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 822                                                    nat_fib_src_low);
 823               m_key.addr = m->local_addr;
 824               m_key.port = m->local_port;
 825               m_key.protocol = m->proto;
 826               m_key.fib_index = local->fib_index;
 827               kv.key = m_key.as_u64;
 828               kv.value = m - sm->static_mappings;
 829               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 830               return 0;
 831             }
 832           else
 833             return VNET_API_ERROR_VALUE_EXIST;
 834         }
 835
 836       if (twice_nat && addr_only)
 837         return VNET_API_ERROR_UNSUPPORTED;
 838
 839       /* Convert VRF id to FIB index */
 840       if (vrf_id != ~0)
 841         fib_index =
 842           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 843                                              nat_fib_src_low);
 844       /* If not specified use inside VRF id from SNAT plugin startup config */
 845       else
 846         {
 847           fib_index = sm->inside_fib_index;
 848           vrf_id = sm->inside_vrf_id;
 849           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 850         }
 851
 852       if (!(out2in_only || identity_nat))
 853         {
 854           m_key.addr = l_addr;
 855           m_key.port = addr_only ? 0 : l_port;
 856           m_key.protocol = addr_only ? 0 : proto;
 857           m_key.fib_index = fib_index;
 858           kv.key = m_key.as_u64;
 859           if (!clib_bihash_search_8_8
 860               (&sm->static_mapping_by_local, &kv, &value))
 861             return VNET_API_ERROR_VALUE_EXIST;
 862         }
 863
 864       /* Find external address in allocated addresses and reserve port for
 865          address and port pair mapping when dynamic translations enabled */
 866       if (!(addr_only || sm->static_mapping_only || out2in_only))
 867         {
 868           for (i = 0; i < vec_len (sm->addresses); i++)
 869             {
 870               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 871                 {
 872                   a = sm->addresses + i;
 873                   /* External port must be unused */
 874                   switch (proto)
 875                     {
 876 #define _(N, j, n, s) \
 877                     case SNAT_PROTOCOL_##N: \
 878                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 879                         return VNET_API_ERROR_INVALID_VALUE; \
 880                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 881                       if (e_port > 1024) \
 882                         { \
 883                           a->busy_##n##_ports++; \
 884                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 885                         } \
 886                       break;
 887                       foreach_snat_protocol
 888 #undef _
 889                     default:
 890                       nat_elog_info ("unknown protocol");
 891                       return VNET_API_ERROR_INVALID_VALUE_2;
 892                     }
 893                   break;
 894                 }
 895             }
 896           /* External address must be allocated */
 897           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 898             {
 899               if (sw_if_index != ~0)
 900                 {
 901                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 902                     {
 903                       rp = sm->to_resolve + i;
 904                       if (rp->addr_only)
 905                         continue;
 906                       if (rp->sw_if_index != sw_if_index &&
 907                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 908                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 909                           rp->e_port != e_port && rp->proto != proto)
 910                         continue;
 911
 912                       vec_del1 (sm->to_resolve, i);
 913                       break;
 914                     }
 915                 }
 916               return VNET_API_ERROR_NO_SUCH_ENTRY;
 917             }
 918         }
 919
 920       pool_get (sm->static_mappings, m);
 921       clib_memset (m, 0, sizeof (*m));
 922       m->tag = vec_dup (tag);
 923       m->local_addr = l_addr;
 924       m->external_addr = e_addr;
 925       m->twice_nat = twice_nat;
 926       if (out2in_only)
 927         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 928       if (addr_only)
 929         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 930       if (identity_nat)
 931         {
 932           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 933           pool_get (m->locals, local);
 934           local->vrf_id = vrf_id;
 935           local->fib_index = fib_index;
 936         }
 937       else
 938         {
 939           m->vrf_id = vrf_id;
 940           m->fib_index = fib_index;
 941         }
 942       if (!addr_only)
 943         {
 944           m->local_port = l_port;
 945           m->external_port = e_port;
 946           m->proto = proto;
 947         }
 948
 949       if (sm->num_workers > 1)
 950         {
 951           ip4_header_t ip = {
 952             .src_address = m->local_addr,
 953           };
 954           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 955           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 956         }
 957       else
 958         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 959
 960       m_key.addr = m->local_addr;
 961       m_key.port = m->local_port;
 962       m_key.protocol = m->proto;
 963       m_key.fib_index = fib_index;
 964       kv.key = m_key.as_u64;
 965       kv.value = m - sm->static_mappings;
 966       if (!out2in_only)
 967         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 968
 969       m_key.addr = m->external_addr;
 970       m_key.port = m->external_port;
 971       m_key.fib_index = 0;
 972       kv.key = m_key.as_u64;
 973       kv.value = m - sm->static_mappings;
 974       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 975
 976       /* Delete dynamic sessions matching local address (+ local port) */
 977       if (!(sm->static_mapping_only))
 978         {
 979           u_key.addr = m->local_addr;
 980           u_key.fib_index = m->fib_index;
 981           kv.key = u_key.as_u64;
 982           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 983             {
 984               user_index = value.value;
 985               u = pool_elt_at_index (tsm->users, user_index);
 986               if (u->nsessions)
 987                 {
 988                   head_index = u->sessions_per_user_list_head_index;
 989                   head = pool_elt_at_index (tsm->list_pool, head_index);
 990                   elt_index = head->next;
 991                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 992                   ses_index = elt->value;
 993                   while (ses_index != ~0)
 994                     {
 995                       s = pool_elt_at_index (tsm->sessions, ses_index);
 996                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 997                       ses_index = elt->value;
 998
 999                       if (snat_is_session_static (s))
1000                         continue;
1001
1002                       if (!addr_only
1003                           && (clib_net_to_host_u16 (s->in2out.port) !=
1004                               m->local_port))
1005                         continue;
1006
1007                       nat_free_session_data (sm, s,
1008                                              tsm - sm->per_thread_data, 0);
1009                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1010
1011                       if (!addr_only && !sm->endpoint_dependent)
1012                         break;
1013                     }
1014                 }
1015             }
1016         }
1017     }
1018   else
1019     {
1020       if (!m)
1021         {
1022           if (sw_if_index != ~0)
1023             return 0;
1024           else
1025             return VNET_API_ERROR_NO_SUCH_ENTRY;
1026         }
1027
1028       if (identity_nat)
1029         {
1030           if (vrf_id == ~0)
1031             vrf_id = sm->inside_vrf_id;
1032
1033           /* *INDENT-OFF* */
1034           pool_foreach (local, m->locals,
1035           ({
1036             if (local->vrf_id == vrf_id)
1037               find = local - m->locals;
1038           }));
1039           /* *INDENT-ON* */
1040           if (find == ~0)
1041             return VNET_API_ERROR_NO_SUCH_ENTRY;
1042
1043           local = pool_elt_at_index (m->locals, find);
1044           fib_index = local->fib_index;
1045           pool_put (m->locals, local);
1046         }
1047       else
1048         fib_index = m->fib_index;
1049
1050       /* Free external address port */
1051       if (!(addr_only || sm->static_mapping_only || out2in_only))
1052         {
1053           for (i = 0; i < vec_len (sm->addresses); i++)
1054             {
1055               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1056                 {
1057                   a = sm->addresses + i;
1058                   switch (proto)
1059                     {
1060 #define _(N, j, n, s) \
1061                     case SNAT_PROTOCOL_##N: \
1062                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1063                       if (e_port > 1024) \
1064                         { \
1065                           a->busy_##n##_ports--; \
1066                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1067                         } \
1068                       break;
1069                       foreach_snat_protocol
1070 #undef _
1071                     default:
1072                       nat_elog_info ("unknown protocol");
1073                       return VNET_API_ERROR_INVALID_VALUE_2;
1074                     }
1075                   break;
1076                 }
1077             }
1078         }
1079
1080       if (sm->num_workers > 1)
1081         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1082       else
1083         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1084
1085       m_key.addr = m->local_addr;
1086       m_key.port = m->local_port;
1087       m_key.protocol = m->proto;
1088       m_key.fib_index = fib_index;
1089       kv.key = m_key.as_u64;
1090       if (!out2in_only)
1091         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1092
1093       /* Delete session(s) for static mapping if exist */
1094       if (!(sm->static_mapping_only) ||
1095           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1096         {
1097           u_key.addr = m->local_addr;
1098           u_key.fib_index = fib_index;
1099           kv.key = u_key.as_u64;
1100           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1101             {
1102               user_index = value.value;
1103               u = pool_elt_at_index (tsm->users, user_index);
1104               if (u->nstaticsessions)
1105                 {
1106                   head_index = u->sessions_per_user_list_head_index;
1107                   head = pool_elt_at_index (tsm->list_pool, head_index);
1108                   elt_index = head->next;
1109                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1110                   ses_index = elt->value;
1111                   while (ses_index != ~0)
1112                     {
1113                       s = pool_elt_at_index (tsm->sessions, ses_index);
1114                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1115                       ses_index = elt->value;
1116
1117                       if (!addr_only)
1118                         {
1119                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1120                               (clib_net_to_host_u16 (s->out2in.port) !=
1121                                e_port))
1122                             continue;
1123                         }
1124
1125                       if (is_lb_session (s))
1126                         continue;
1127
1128                       if (!snat_is_session_static (s))
1129                         continue;
1130
1131                       nat_free_session_data (sm, s,
1132                                              tsm - sm->per_thread_data, 0);
1133                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1134
1135                       if (!addr_only && !sm->endpoint_dependent)
1136                         break;
1137                     }
1138                 }
1139             }
1140         }
1141
1142       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1143       if (pool_elts (m->locals))
1144         return 0;
1145
1146       m_key.addr = m->external_addr;
1147       m_key.port = m->external_port;
1148       m_key.fib_index = 0;
1149       kv.key = m_key.as_u64;
1150       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1151
1152       vec_free (m->tag);
1153       vec_free (m->workers);
1154       /* Delete static mapping from pool */
1155       pool_put (sm->static_mappings, m);
1156     }
1157
1158   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1159     return 0;
1160
1161   /* Add/delete external address to FIB */
1162   /* *INDENT-OFF* */
1163   pool_foreach (interface, sm->interfaces,
1164   ({
1165     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1166       continue;
1167
1168     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1169     break;
1170   }));
1171   pool_foreach (interface, sm->output_feature_interfaces,
1172   ({
1173     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1174       continue;
1175
1176     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1177     break;
1178   }));
1179   /* *INDENT-ON* */
1180
1181   return 0;
1182 }
1183
1184 int
1185 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1186                                  snat_protocol_t proto,
1187                                  nat44_lb_addr_port_t * locals, u8 is_add,
1188                                  twice_nat_type_t twice_nat, u8 out2in_only,
1189                                  u8 * tag, u32 affinity)
1190 {
1191   snat_main_t *sm = &snat_main;
1192   snat_static_mapping_t *m;
1193   snat_session_key_t m_key;
1194   clib_bihash_kv_8_8_t kv, value;
1195   snat_address_t *a = 0;
1196   int i;
1197   nat44_lb_addr_port_t *local;
1198   u32 elt_index, head_index, ses_index;
1199   snat_main_per_thread_data_t *tsm;
1200   snat_user_key_t u_key;
1201   snat_user_t *u;
1202   snat_session_t *s;
1203   dlist_elt_t *head, *elt;
1204   uword *bitmap = 0;
1205
1206   if (!sm->endpoint_dependent)
1207     return VNET_API_ERROR_FEATURE_DISABLED;
1208
1209   m_key.addr = e_addr;
1210   m_key.port = e_port;
1211   m_key.protocol = proto;
1212   m_key.fib_index = 0;
1213   kv.key = m_key.as_u64;
1214   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1215     m = 0;
1216   else
1217     m = pool_elt_at_index (sm->static_mappings, value.value);
1218
1219   if (is_add)
1220     {
1221       if (m)
1222         return VNET_API_ERROR_VALUE_EXIST;
1223
1224       if (vec_len (locals) < 2)
1225         return VNET_API_ERROR_INVALID_VALUE;
1226
1227       /* Find external address in allocated addresses and reserve port for
1228          address and port pair mapping when dynamic translations enabled */
1229       if (!(sm->static_mapping_only || out2in_only))
1230         {
1231           for (i = 0; i < vec_len (sm->addresses); i++)
1232             {
1233               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1234                 {
1235                   a = sm->addresses + i;
1236                   /* External port must be unused */
1237                   switch (proto)
1238                     {
1239 #define _(N, j, n, s) \
1240                     case SNAT_PROTOCOL_##N: \
1241                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1242                         return VNET_API_ERROR_INVALID_VALUE; \
1243                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1244                       if (e_port > 1024) \
1245                         { \
1246                           a->busy_##n##_ports++; \
1247                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1248                         } \
1249                       break;
1250                       foreach_snat_protocol
1251 #undef _
1252                     default:
1253                       nat_elog_info ("unknown protocol");
1254                       return VNET_API_ERROR_INVALID_VALUE_2;
1255                     }
1256                   break;
1257                 }
1258             }
1259           /* External address must be allocated */
1260           if (!a)
1261             return VNET_API_ERROR_NO_SUCH_ENTRY;
1262         }
1263
1264       pool_get (sm->static_mappings, m);
1265       clib_memset (m, 0, sizeof (*m));
1266       m->tag = vec_dup (tag);
1267       m->external_addr = e_addr;
1268       m->external_port = e_port;
1269       m->proto = proto;
1270       m->twice_nat = twice_nat;
1271       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1272       if (out2in_only)
1273         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1274       m->affinity = affinity;
1275
1276       if (affinity)
1277         m->affinity_per_service_list_head_index =
1278           nat_affinity_get_per_service_list_head_index ();
1279       else
1280         m->affinity_per_service_list_head_index = ~0;
1281
1282       m_key.addr = m->external_addr;
1283       m_key.port = m->external_port;
1284       m_key.protocol = m->proto;
1285       m_key.fib_index = 0;
1286       kv.key = m_key.as_u64;
1287       kv.value = m - sm->static_mappings;
1288       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1289         {
1290           nat_elog_err ("static_mapping_by_external key add failed");
1291           return VNET_API_ERROR_UNSPECIFIED;
1292         }
1293
1294       m_key.fib_index = m->fib_index;
1295       for (i = 0; i < vec_len (locals); i++)
1296         {
1297           locals[i].fib_index =
1298             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1299                                                locals[i].vrf_id,
1300                                                nat_fib_src_low);
1301           m_key.addr = locals[i].addr;
1302           m_key.fib_index = locals[i].fib_index;
1303           if (!out2in_only)
1304             {
1305               m_key.port = locals[i].port;
1306               kv.key = m_key.as_u64;
1307               kv.value = m - sm->static_mappings;
1308               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1309             }
1310           locals[i].prefix = (i == 0) ? locals[i].probability :
1311             (locals[i - 1].prefix + locals[i].probability);
1312           pool_get (m->locals, local);
1313           *local = locals[i];
1314           if (sm->num_workers > 1)
1315             {
1316               ip4_header_t ip = {
1317                 .src_address = locals[i].addr,
1318               };
1319               bitmap =
1320                 clib_bitmap_set (bitmap,
1321                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1322                                  1);
1323             }
1324         }
1325
1326       /* Assign workers */
1327       if (sm->num_workers > 1)
1328         {
1329           /* *INDENT-OFF* */
1330           clib_bitmap_foreach (i, bitmap,
1331             ({
1332                vec_add1(m->workers, i);
1333             }));
1334           /* *INDENT-ON* */
1335         }
1336     }
1337   else
1338     {
1339       if (!m)
1340         return VNET_API_ERROR_NO_SUCH_ENTRY;
1341
1342       if (!is_lb_static_mapping (m))
1343         return VNET_API_ERROR_INVALID_VALUE;
1344
1345       /* Free external address port */
1346       if (!(sm->static_mapping_only || out2in_only))
1347         {
1348           for (i = 0; i < vec_len (sm->addresses); i++)
1349             {
1350               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1351                 {
1352                   a = sm->addresses + i;
1353                   switch (proto)
1354                     {
1355 #define _(N, j, n, s) \
1356                     case SNAT_PROTOCOL_##N: \
1357                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1358                       if (e_port > 1024) \
1359                         { \
1360                           a->busy_##n##_ports--; \
1361                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1362                         } \
1363                       break;
1364                       foreach_snat_protocol
1365 #undef _
1366                     default:
1367                       nat_elog_info ("unknown protocol");
1368                       return VNET_API_ERROR_INVALID_VALUE_2;
1369                     }
1370                   break;
1371                 }
1372             }
1373         }
1374
1375       m_key.addr = m->external_addr;
1376       m_key.port = m->external_port;
1377       m_key.protocol = m->proto;
1378       m_key.fib_index = 0;
1379       kv.key = m_key.as_u64;
1380       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1381         {
1382           nat_elog_err ("static_mapping_by_external key del failed");
1383           return VNET_API_ERROR_UNSPECIFIED;
1384         }
1385
1386       /* *INDENT-OFF* */
1387       pool_foreach (local, m->locals,
1388       ({
1389           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1390                             nat_fib_src_low);
1391           m_key.addr = local->addr;
1392           if (!out2in_only)
1393             {
1394               m_key.port = local->port;
1395               m_key.fib_index = local->fib_index;
1396               kv.key = m_key.as_u64;
1397               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1398                 {
1399                   nat_elog_err ("static_mapping_by_local key del failed");
1400                   return VNET_API_ERROR_UNSPECIFIED;
1401                 }
1402             }
1403
1404           if (sm->num_workers > 1)
1405             {
1406               ip4_header_t ip = {
1407                 .src_address = local->addr,
1408               };
1409               tsm = vec_elt_at_index (sm->per_thread_data,
1410                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1411             }
1412           else
1413             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1414
1415           /* Delete sessions */
1416           u_key.addr = local->addr;
1417           u_key.fib_index = local->fib_index;
1418           kv.key = u_key.as_u64;
1419           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1420             {
1421               u = pool_elt_at_index (tsm->users, value.value);
1422               if (u->nstaticsessions)
1423                 {
1424                   head_index = u->sessions_per_user_list_head_index;
1425                   head = pool_elt_at_index (tsm->list_pool, head_index);
1426                   elt_index = head->next;
1427                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1428                   ses_index = elt->value;
1429                   while (ses_index != ~0)
1430                     {
1431                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1432                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1433                       ses_index = elt->value;
1434
1435                       if (!(is_lb_session (s)))
1436                         continue;
1437
1438                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1439                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1440                         continue;
1441
1442                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1443                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1444                     }
1445                 }
1446             }
1447       }));
1448       /* *INDENT-ON* */
1449       if (m->affinity)
1450         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1451       pool_free (m->locals);
1452       vec_free (m->tag);
1453       vec_free (m->workers);
1454
1455       pool_put (sm->static_mappings, m);
1456     }
1457
1458   return 0;
1459 }
1460
1461 int
1462 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1463                                        ip4_address_t l_addr, u16 l_port,
1464                                        snat_protocol_t proto, u32 vrf_id,
1465                                        u8 probability, u8 is_add)
1466 {
1467   snat_main_t *sm = &snat_main;
1468   snat_static_mapping_t *m = 0;
1469   snat_session_key_t m_key;
1470   clib_bihash_kv_8_8_t kv, value;
1471   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1472   snat_main_per_thread_data_t *tsm;
1473   snat_user_key_t u_key;
1474   snat_user_t *u;
1475   snat_session_t *s;
1476   dlist_elt_t *head, *elt;
1477   u32 elt_index, head_index, ses_index, *locals = 0;
1478   uword *bitmap = 0;
1479   int i;
1480
1481   if (!sm->endpoint_dependent)
1482     return VNET_API_ERROR_FEATURE_DISABLED;
1483
1484   m_key.addr = e_addr;
1485   m_key.port = e_port;
1486   m_key.protocol = proto;
1487   m_key.fib_index = 0;
1488   kv.key = m_key.as_u64;
1489   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1490     m = pool_elt_at_index (sm->static_mappings, value.value);
1491
1492   if (!m)
1493     return VNET_API_ERROR_NO_SUCH_ENTRY;
1494
1495   if (!is_lb_static_mapping (m))
1496     return VNET_API_ERROR_INVALID_VALUE;
1497
1498   /* *INDENT-OFF* */
1499   pool_foreach (local, m->locals,
1500   ({
1501     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1502         (local->vrf_id == vrf_id))
1503       {
1504         match_local = local;
1505         break;
1506       }
1507   }));
1508   /* *INDENT-ON* */
1509
1510   if (is_add)
1511     {
1512       if (match_local)
1513         return VNET_API_ERROR_VALUE_EXIST;
1514
1515       pool_get (m->locals, local);
1516       clib_memset (local, 0, sizeof (*local));
1517       local->addr.as_u32 = l_addr.as_u32;
1518       local->port = l_port;
1519       local->probability = probability;
1520       local->vrf_id = vrf_id;
1521       local->fib_index =
1522         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1523                                            nat_fib_src_low);
1524
1525       if (!is_out2in_only_static_mapping (m))
1526         {
1527           m_key.addr = l_addr;
1528           m_key.port = l_port;
1529           m_key.fib_index = local->fib_index;
1530           kv.key = m_key.as_u64;
1531           kv.value = m - sm->static_mappings;
1532           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1533             nat_elog_err ("static_mapping_by_local key add failed");
1534         }
1535     }
1536   else
1537     {
1538       if (!match_local)
1539         return VNET_API_ERROR_NO_SUCH_ENTRY;
1540
1541       if (pool_elts (m->locals) < 3)
1542         return VNET_API_ERROR_UNSPECIFIED;
1543
1544       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1545                         nat_fib_src_low);
1546
1547       if (!is_out2in_only_static_mapping (m))
1548         {
1549           m_key.addr = l_addr;
1550           m_key.port = l_port;
1551           m_key.fib_index = match_local->fib_index;
1552           kv.key = m_key.as_u64;
1553           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1554             nat_elog_err ("static_mapping_by_local key del failed");
1555         }
1556
1557       if (sm->num_workers > 1)
1558         {
1559           ip4_header_t ip = {
1560             .src_address = local->addr,
1561           };
1562           tsm = vec_elt_at_index (sm->per_thread_data,
1563                                   sm->worker_in2out_cb (&ip, m->fib_index,
1564                                                         0));
1565         }
1566       else
1567         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1568
1569       /* Delete sessions */
1570       u_key.addr = match_local->addr;
1571       u_key.fib_index = match_local->fib_index;
1572       kv.key = u_key.as_u64;
1573       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1574         {
1575           u = pool_elt_at_index (tsm->users, value.value);
1576           if (u->nstaticsessions)
1577             {
1578               head_index = u->sessions_per_user_list_head_index;
1579               head = pool_elt_at_index (tsm->list_pool, head_index);
1580               elt_index = head->next;
1581               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1582               ses_index = elt->value;
1583               while (ses_index != ~0)
1584                 {
1585                   s = pool_elt_at_index (tsm->sessions, ses_index);
1586                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1587                   ses_index = elt->value;
1588
1589                   if (!(is_lb_session (s)))
1590                     continue;
1591
1592                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1593                       (clib_net_to_host_u16 (s->in2out.port) !=
1594                        match_local->port))
1595                     continue;
1596
1597                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1598                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1599                 }
1600             }
1601         }
1602
1603       pool_put (m->locals, match_local);
1604     }
1605
1606   vec_free (m->workers);
1607
1608   /* *INDENT-OFF* */
1609   pool_foreach (local, m->locals,
1610   ({
1611     vec_add1 (locals, local - m->locals);
1612     if (sm->num_workers > 1)
1613       {
1614         ip4_header_t ip;
1615         ip.src_address.as_u32 = local->addr.as_u32,
1616         bitmap = clib_bitmap_set (bitmap,
1617                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1618                                   1);
1619       }
1620   }));
1621   /* *INDENT-ON* */
1622
1623   ASSERT (vec_len (locals) > 1);
1624
1625   local = pool_elt_at_index (m->locals, locals[0]);
1626   local->prefix = local->probability;
1627   for (i = 1; i < vec_len (locals); i++)
1628     {
1629       local = pool_elt_at_index (m->locals, locals[i]);
1630       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1631       local->prefix = local->probability + prev_local->prefix;
1632     }
1633
1634   /* Assign workers */
1635   if (sm->num_workers > 1)
1636     {
1637       /* *INDENT-OFF* */
1638       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1639       /* *INDENT-ON* */
1640     }
1641
1642   return 0;
1643 }
1644
1645 int
1646 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1647                   u8 twice_nat)
1648 {
1649   snat_address_t *a = 0;
1650   snat_session_t *ses;
1651   u32 *ses_to_be_removed = 0, *ses_index;
1652   snat_main_per_thread_data_t *tsm;
1653   snat_static_mapping_t *m;
1654   snat_interface_t *interface;
1655   int i;
1656   snat_address_t *addresses =
1657     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1658
1659   /* Find SNAT address */
1660   for (i = 0; i < vec_len (addresses); i++)
1661     {
1662       if (addresses[i].addr.as_u32 == addr.as_u32)
1663         {
1664           a = addresses + i;
1665           break;
1666         }
1667     }
1668   if (!a)
1669     return VNET_API_ERROR_NO_SUCH_ENTRY;
1670
1671   if (delete_sm)
1672     {
1673       /* *INDENT-OFF* */
1674       pool_foreach (m, sm->static_mappings,
1675       ({
1676           if (m->external_addr.as_u32 == addr.as_u32)
1677             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1678                                             m->local_port, m->external_port,
1679                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1680                                             m->proto, 0, m->twice_nat,
1681                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1682       }));
1683       /* *INDENT-ON* */
1684     }
1685   else
1686     {
1687       /* Check if address is used in some static mapping */
1688       if (is_snat_address_used_in_static_mapping (sm, addr))
1689         {
1690           nat_elog_notice ("address used in static mapping");
1691           return VNET_API_ERROR_UNSPECIFIED;
1692         }
1693     }
1694
1695   if (a->fib_index != ~0)
1696     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1697
1698   /* Delete sessions using address */
1699   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1700     {
1701       /* *INDENT-OFF* */
1702       vec_foreach (tsm, sm->per_thread_data)
1703         {
1704           pool_foreach (ses, tsm->sessions, ({
1705             if (ses->out2in.addr.as_u32 == addr.as_u32)
1706               {
1707                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1708                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1709               }
1710           }));
1711
1712           vec_foreach (ses_index, ses_to_be_removed)
1713             {
1714               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1715               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1716             }
1717
1718           vec_free (ses_to_be_removed);
1719         }
1720       /* *INDENT-ON* */
1721     }
1722
1723 #define _(N, i, n, s) \
1724   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1725   vec_free (a->busy_##n##_ports_per_thread);
1726   foreach_snat_protocol
1727 #undef _
1728     if (twice_nat)
1729     {
1730       vec_del1 (sm->twice_nat_addresses, i);
1731       return 0;
1732     }
1733   else
1734     vec_del1 (sm->addresses, i);
1735
1736   /* Delete external address from FIB */
1737   /* *INDENT-OFF* */
1738   pool_foreach (interface, sm->interfaces,
1739   ({
1740     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1741       continue;
1742
1743     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1744     break;
1745   }));
1746   pool_foreach (interface, sm->output_feature_interfaces,
1747   ({
1748     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1749       continue;
1750
1751     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1752     break;
1753   }));
1754   /* *INDENT-ON* */
1755
1756   return 0;
1757 }
1758
1759 int
1760 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1761 {
1762   snat_main_t *sm = &snat_main;
1763   snat_interface_t *i;
1764   const char *feature_name, *del_feature_name;
1765   snat_address_t *ap;
1766   snat_static_mapping_t *m;
1767   snat_det_map_t *dm;
1768   nat_outside_fib_t *outside_fib;
1769   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1770                                                        sw_if_index);
1771
1772   if (sm->out2in_dpo && !is_inside)
1773     return VNET_API_ERROR_UNSUPPORTED;
1774
1775   /* *INDENT-OFF* */
1776   pool_foreach (i, sm->output_feature_interfaces,
1777   ({
1778     if (i->sw_if_index == sw_if_index)
1779       return VNET_API_ERROR_VALUE_EXIST;
1780   }));
1781   /* *INDENT-ON* */
1782
1783   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1784     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1785   else
1786     {
1787       if (sm->num_workers > 1 && !sm->deterministic)
1788         feature_name =
1789           is_inside ? "nat44-in2out-worker-handoff" :
1790           "nat44-out2in-worker-handoff";
1791       else if (sm->deterministic)
1792         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1793       else if (sm->endpoint_dependent)
1794         {
1795           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1796         }
1797       else
1798         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1799     }
1800
1801   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1802     sm->fq_in2out_index =
1803       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1804
1805   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1806     sm->fq_out2in_index =
1807       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1808
1809   if (!is_inside)
1810     {
1811       /* *INDENT-OFF* */
1812       vec_foreach (outside_fib, sm->outside_fibs)
1813         {
1814           if (outside_fib->fib_index == fib_index)
1815             {
1816               if (is_del)
1817                 {
1818                   outside_fib->refcount--;
1819                   if (!outside_fib->refcount)
1820                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1821                 }
1822               else
1823                 outside_fib->refcount++;
1824               goto feature_set;
1825             }
1826         }
1827       /* *INDENT-ON* */
1828       if (!is_del)
1829         {
1830           vec_add2 (sm->outside_fibs, outside_fib, 1);
1831           outside_fib->refcount = 1;
1832           outside_fib->fib_index = fib_index;
1833         }
1834     }
1835 feature_set:
1836   /* *INDENT-OFF* */
1837   pool_foreach (i, sm->interfaces,
1838   ({
1839     if (i->sw_if_index == sw_if_index)
1840       {
1841         if (is_del)
1842           {
1843             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1844               {
1845                 if (is_inside)
1846                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1847                 else
1848                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1849
1850                 if (sm->num_workers > 1 && !sm->deterministic)
1851                   {
1852                     del_feature_name = "nat44-handoff-classify";
1853                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1854                                                  "nat44-out2in-worker-handoff";
1855                   }
1856                 else if (sm->deterministic)
1857                   {
1858                     del_feature_name = "nat44-det-classify";
1859                     feature_name = !is_inside ?  "nat44-det-in2out" :
1860                                                  "nat44-det-out2in";
1861                   }
1862                 else if (sm->endpoint_dependent)
1863                   {
1864                     del_feature_name = "nat44-ed-classify";
1865                     feature_name = !is_inside ?  "nat-pre-in2out" :
1866                                                  "nat-pre-out2in";
1867                   }
1868                 else
1869                   {
1870                     del_feature_name = "nat44-classify";
1871                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1872                   }
1873
1874                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1875                 if (rv)
1876                   return rv;
1877                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1878                                              sw_if_index, 0, 0, 0);
1879                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1880                                              sw_if_index, 1, 0, 0);
1881                 if (!is_inside)
1882                   {
1883                     if (sm->endpoint_dependent)
1884                       vnet_feature_enable_disable ("ip4-local",
1885                                                    "nat44-ed-hairpinning",
1886                                                    sw_if_index, 1, 0, 0);
1887                     else if (!sm->deterministic)
1888                       vnet_feature_enable_disable ("ip4-local",
1889                                                    "nat44-hairpinning",
1890                                                    sw_if_index, 1, 0, 0);
1891                   }
1892               }
1893             else
1894               {
1895                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1896                 if (rv)
1897                   return rv;
1898                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1899                                              sw_if_index, 0, 0, 0);
1900                 pool_put (sm->interfaces, i);
1901                 if (is_inside)
1902                   {
1903                     if (sm->endpoint_dependent)
1904                       vnet_feature_enable_disable ("ip4-local",
1905                                                    "nat44-ed-hairpinning",
1906                                                    sw_if_index, 0, 0, 0);
1907                     else if (!sm->deterministic)
1908                       vnet_feature_enable_disable ("ip4-local",
1909                                                    "nat44-hairpinning",
1910                                                    sw_if_index, 0, 0, 0);
1911                   }
1912               }
1913           }
1914         else
1915           {
1916             if ((nat_interface_is_inside(i) && is_inside) ||
1917                 (nat_interface_is_outside(i) && !is_inside))
1918               return 0;
1919
1920             if (sm->num_workers > 1 && !sm->deterministic)
1921               {
1922                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1923                                                  "nat44-out2in-worker-handoff";
1924                 feature_name = "nat44-handoff-classify";
1925               }
1926             else if (sm->deterministic)
1927               {
1928                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1929                                                  "nat44-det-out2in";
1930                 feature_name = "nat44-det-classify";
1931               }
1932             else if (sm->endpoint_dependent)
1933               {
1934                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
1935                                                  "nat-pre-out2in";
1936
1937                 feature_name = "nat44-ed-classify";
1938               }
1939             else
1940               {
1941                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1942                 feature_name = "nat44-classify";
1943               }
1944
1945             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1946             if (rv)
1947               return rv;
1948             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1949                                          sw_if_index, 0, 0, 0);
1950             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1951                                          sw_if_index, 1, 0, 0);
1952             if (!is_inside)
1953               {
1954                 if (sm->endpoint_dependent)
1955                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1956                                                sw_if_index, 0, 0, 0);
1957                 else if (!sm->deterministic)
1958                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1959                                                sw_if_index, 0, 0, 0);
1960               }
1961             goto set_flags;
1962           }
1963
1964         goto fib;
1965       }
1966   }));
1967   /* *INDENT-ON* */
1968
1969   if (is_del)
1970     return VNET_API_ERROR_NO_SUCH_ENTRY;
1971
1972   pool_get (sm->interfaces, i);
1973   i->sw_if_index = sw_if_index;
1974   i->flags = 0;
1975   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1976                                0);
1977
1978   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1979   if (rv)
1980     return rv;
1981
1982   if (is_inside && !sm->out2in_dpo)
1983     {
1984       if (sm->endpoint_dependent)
1985         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1986                                      sw_if_index, 1, 0, 0);
1987       else if (!sm->deterministic)
1988         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1989                                      sw_if_index, 1, 0, 0);
1990     }
1991
1992 set_flags:
1993   if (is_inside)
1994     {
1995       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1996       return 0;
1997     }
1998   else
1999     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2000
2001   /* Add/delete external addresses to FIB */
2002 fib:
2003   /* *INDENT-OFF* */
2004   vec_foreach (ap, sm->addresses)
2005     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2006
2007   pool_foreach (m, sm->static_mappings,
2008   ({
2009     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2010       continue;
2011
2012     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2013   }));
2014
2015   pool_foreach (dm, sm->det_maps,
2016   ({
2017     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2018   }));
2019   /* *INDENT-ON* */
2020
2021   return 0;
2022 }
2023
2024 int
2025 snat_interface_add_del_output_feature (u32 sw_if_index,
2026                                        u8 is_inside, int is_del)
2027 {
2028   snat_main_t *sm = &snat_main;
2029   snat_interface_t *i;
2030   snat_address_t *ap;
2031   snat_static_mapping_t *m;
2032   nat_outside_fib_t *outside_fib;
2033   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2034                                                        sw_if_index);
2035
2036
2037   if (sm->deterministic ||
2038       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2039     return VNET_API_ERROR_UNSUPPORTED;
2040
2041   /* *INDENT-OFF* */
2042   pool_foreach (i, sm->interfaces,
2043   ({
2044     if (i->sw_if_index == sw_if_index)
2045       return VNET_API_ERROR_VALUE_EXIST;
2046   }));
2047   /* *INDENT-ON* */
2048
2049   if (!is_inside)
2050     {
2051       /* *INDENT-OFF* */
2052       vec_foreach (outside_fib, sm->outside_fibs)
2053         {
2054           if (outside_fib->fib_index == fib_index)
2055             {
2056               if (is_del)
2057                 {
2058                   outside_fib->refcount--;
2059                   if (!outside_fib->refcount)
2060                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2061                 }
2062               else
2063                 outside_fib->refcount++;
2064               goto feature_set;
2065             }
2066         }
2067       /* *INDENT-ON* */
2068       if (!is_del)
2069         {
2070           vec_add2 (sm->outside_fibs, outside_fib, 1);
2071           outside_fib->refcount = 1;
2072           outside_fib->fib_index = fib_index;
2073         }
2074     }
2075
2076 feature_set:
2077   if (is_inside)
2078     {
2079       if (sm->endpoint_dependent)
2080         {
2081           int rv =
2082             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2083           if (rv)
2084             return rv;
2085           rv =
2086             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2087                                                             !is_del);
2088           if (rv)
2089             return rv;
2090           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2091                                        sw_if_index, !is_del, 0, 0);
2092           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2093                                        sw_if_index, !is_del, 0, 0);
2094         }
2095       else
2096         {
2097           int rv =
2098             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2099           if (rv)
2100             return rv;
2101           rv =
2102             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2103                                                             !is_del);
2104           if (rv)
2105             return rv;
2106           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2107                                        sw_if_index, !is_del, 0, 0);
2108           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2109                                        sw_if_index, !is_del, 0, 0);
2110         }
2111       goto fq;
2112     }
2113
2114   if (sm->num_workers > 1)
2115     {
2116       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2117       if (rv)
2118         return rv;
2119       rv =
2120         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2121       if (rv)
2122         return rv;
2123       vnet_feature_enable_disable ("ip4-unicast",
2124                                    "nat44-out2in-worker-handoff",
2125                                    sw_if_index, !is_del, 0, 0);
2126       vnet_feature_enable_disable ("ip4-output",
2127                                    "nat44-in2out-output-worker-handoff",
2128                                    sw_if_index, !is_del, 0, 0);
2129     }
2130   else
2131     {
2132       if (sm->endpoint_dependent)
2133         {
2134           int rv =
2135             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2136           if (rv)
2137             return rv;
2138           rv =
2139             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2140                                                             !is_del);
2141           if (rv)
2142             return rv;
2143           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2144                                        sw_if_index, !is_del, 0, 0);
2145           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2146                                        sw_if_index, !is_del, 0, 0);
2147         }
2148       else
2149         {
2150           int rv =
2151             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2152           if (rv)
2153             return rv;
2154           rv =
2155             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2156                                                             !is_del);
2157           if (rv)
2158             return rv;
2159           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2160                                        sw_if_index, !is_del, 0, 0);
2161           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2162                                        sw_if_index, !is_del, 0, 0);
2163         }
2164     }
2165
2166 fq:
2167   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2168     sm->fq_in2out_output_index =
2169       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2170
2171   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2172     sm->fq_out2in_index =
2173       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2174
2175   /* *INDENT-OFF* */
2176   pool_foreach (i, sm->output_feature_interfaces,
2177   ({
2178     if (i->sw_if_index == sw_if_index)
2179       {
2180         if (is_del)
2181           pool_put (sm->output_feature_interfaces, i);
2182         else
2183           return VNET_API_ERROR_VALUE_EXIST;
2184
2185         goto fib;
2186       }
2187   }));
2188   /* *INDENT-ON* */
2189
2190   if (is_del)
2191     return VNET_API_ERROR_NO_SUCH_ENTRY;
2192
2193   pool_get (sm->output_feature_interfaces, i);
2194   i->sw_if_index = sw_if_index;
2195   i->flags = 0;
2196   if (is_inside)
2197     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2198   else
2199     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2200
2201   /* Add/delete external addresses to FIB */
2202 fib:
2203   if (is_inside)
2204     return 0;
2205
2206   /* *INDENT-OFF* */
2207   vec_foreach (ap, sm->addresses)
2208     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2209
2210   pool_foreach (m, sm->static_mappings,
2211   ({
2212     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2213       continue;
2214
2215     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2216   }));
2217   /* *INDENT-ON* */
2218
2219   return 0;
2220 }
2221
2222 int
2223 snat_set_workers (uword * bitmap)
2224 {
2225   snat_main_t *sm = &snat_main;
2226   int i, j = 0;
2227
2228   if (sm->num_workers < 2)
2229     return VNET_API_ERROR_FEATURE_DISABLED;
2230
2231   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2232     return VNET_API_ERROR_INVALID_WORKER;
2233
2234   vec_free (sm->workers);
2235   /* *INDENT-OFF* */
2236   clib_bitmap_foreach (i, bitmap,
2237     ({
2238       vec_add1(sm->workers, i);
2239       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2240       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2241       j++;
2242     }));
2243   /* *INDENT-ON* */
2244
2245   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2246   sm->num_snat_thread = _vec_len (sm->workers);
2247
2248   return 0;
2249 }
2250
2251 static void
2252 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2253                          u32 old_fib_index)
2254 {
2255   snat_main_t *sm = &snat_main;
2256   nat_outside_fib_t *outside_fib;
2257   snat_interface_t *i;
2258   u8 is_add = 1;
2259   u8 match = 0;
2260
2261   if (new_fib_index == old_fib_index)
2262     return;
2263
2264   if (!vec_len (sm->outside_fibs))
2265     return;
2266
2267   /* *INDENT-OFF* */
2268   pool_foreach (i, sm->interfaces,
2269     ({
2270       if (i->sw_if_index == sw_if_index)
2271         {
2272           if (!(nat_interface_is_outside (i)))
2273             return;
2274           match = 1;
2275         }
2276     }));
2277
2278   pool_foreach (i, sm->output_feature_interfaces,
2279     ({
2280       if (i->sw_if_index == sw_if_index)
2281         {
2282           if (!(nat_interface_is_outside (i)))
2283             return;
2284           match = 1;
2285         }
2286     }));
2287   /* *INDENT-ON* */
2288
2289   if (!match)
2290     return;
2291
2292   vec_foreach (outside_fib, sm->outside_fibs)
2293   {
2294     if (outside_fib->fib_index == old_fib_index)
2295       {
2296         outside_fib->refcount--;
2297         if (!outside_fib->refcount)
2298           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2299         break;
2300       }
2301   }
2302
2303   vec_foreach (outside_fib, sm->outside_fibs)
2304   {
2305     if (outside_fib->fib_index == new_fib_index)
2306       {
2307         outside_fib->refcount++;
2308         is_add = 0;
2309         break;
2310       }
2311   }
2312
2313   if (is_add)
2314     {
2315       vec_add2 (sm->outside_fibs, outside_fib, 1);
2316       outside_fib->refcount = 1;
2317       outside_fib->fib_index = new_fib_index;
2318     }
2319 }
2320
2321 static void
2322 snat_ip4_table_bind (ip4_main_t * im,
2323                      uword opaque,
2324                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2325 {
2326   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2327 }
2328
2329 static void
2330 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2331                                        uword opaque,
2332                                        u32 sw_if_index,
2333                                        ip4_address_t * address,
2334                                        u32 address_length,
2335                                        u32 if_address_index, u32 is_delete);
2336
2337 static void
2338 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2339                                  uword opaque,
2340                                  u32 sw_if_index,
2341                                  ip4_address_t * address,
2342                                  u32 address_length,
2343                                  u32 if_address_index, u32 is_delete);
2344
2345 static int
2346 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2347                                  u32 fib_index,
2348                                  u32 thread_index,
2349                                  snat_session_key_t * k,
2350                                  u16 port_per_thread, u32 snat_thread_index);
2351
2352 static clib_error_t *
2353 snat_init (vlib_main_t * vm)
2354 {
2355   snat_main_t *sm = &snat_main;
2356   clib_error_t *error = 0;
2357   ip4_main_t *im = &ip4_main;
2358   ip_lookup_main_t *lm = &im->lookup_main;
2359   uword *p;
2360   vlib_thread_registration_t *tr;
2361   vlib_thread_main_t *tm = vlib_get_thread_main ();
2362   uword *bitmap = 0;
2363   u32 i;
2364   ip4_add_del_interface_address_callback_t cb4;
2365   vlib_node_t *node;
2366
2367   sm->vlib_main = vm;
2368   sm->vnet_main = vnet_get_main ();
2369   sm->ip4_main = im;
2370   sm->ip4_lookup_main = lm;
2371   sm->api_main = vlibapi_get_main ();
2372   sm->first_worker_index = 0;
2373   sm->num_workers = 0;
2374   sm->num_snat_thread = 1;
2375   sm->workers = 0;
2376   sm->port_per_thread = 0xffff - 1024;
2377   sm->fq_in2out_index = ~0;
2378   sm->fq_in2out_output_index = ~0;
2379   sm->fq_out2in_index = ~0;
2380
2381
2382   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2383   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2384   sm->forwarding_enabled = 0;
2385   sm->log_class = vlib_log_register_class ("nat", 0);
2386   sm->log_level = SNAT_LOG_ERROR;
2387   sm->mss_clamping = 0;
2388
2389   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2390   sm->error_node_index = node->index;
2391
2392   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2393   sm->pre_in2out_node_index = node->index;
2394   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2395   sm->pre_out2in_node_index = node->index;
2396
2397   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2398   sm->pre_in2out_node_index = node->index;
2399
2400   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2401   sm->pre_out2in_node_index = node->index;
2402
2403   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2404   sm->in2out_node_index = node->index;
2405   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2406   sm->in2out_output_node_index = node->index;
2407   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2408   sm->in2out_fast_node_index = node->index;
2409   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2410   sm->in2out_slowpath_node_index = node->index;
2411   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2412   sm->in2out_slowpath_output_node_index = node->index;
2413
2414   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2415   sm->ed_in2out_node_index = node->index;
2416   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2417   sm->ed_in2out_slowpath_node_index = node->index;
2418
2419   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2420   sm->out2in_node_index = node->index;
2421   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2422   sm->out2in_fast_node_index = node->index;
2423
2424   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2425   sm->ed_out2in_node_index = node->index;
2426   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2427   sm->ed_out2in_slowpath_node_index = node->index;
2428
2429   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2430   sm->det_in2out_node_index = node->index;
2431   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2432   sm->det_out2in_node_index = node->index;
2433
2434   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2435   sm->hairpinning_node_index = node->index;
2436   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2437   sm->hairpin_dst_node_index = node->index;
2438   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2439   sm->hairpin_src_node_index = node->index;
2440   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2441   sm->ed_hairpinning_node_index = node->index;
2442   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2443   sm->ed_hairpin_dst_node_index = node->index;
2444   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2445   sm->ed_hairpin_src_node_index = node->index;
2446
2447   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2448   if (p)
2449     {
2450       tr = (vlib_thread_registration_t *) p[0];
2451       if (tr)
2452         {
2453           sm->num_workers = tr->count;
2454           sm->first_worker_index = tr->first_index;
2455         }
2456     }
2457
2458   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2459
2460   /* Use all available workers by default */
2461   if (sm->num_workers > 1)
2462     {
2463       for (i = 0; i < sm->num_workers; i++)
2464         bitmap = clib_bitmap_set (bitmap, i, 1);
2465       snat_set_workers (bitmap);
2466       clib_bitmap_free (bitmap);
2467     }
2468   else
2469     {
2470       sm->per_thread_data[0].snat_thread_index = 0;
2471     }
2472
2473   error = snat_api_init (vm, sm);
2474   if (error)
2475     return error;
2476
2477   /* Set up the interface address add/del callback */
2478   cb4.function = snat_ip4_add_del_interface_address_cb;
2479   cb4.function_opaque = 0;
2480
2481   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2482
2483   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2484   cb4.function_opaque = 0;
2485
2486   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2487
2488   nat_dpo_module_init ();
2489
2490   /* Init counters */
2491   sm->total_users.name = "total-users";
2492   sm->total_users.stat_segment_name = "/nat44/total-users";
2493   vlib_validate_simple_counter (&sm->total_users, 0);
2494   vlib_zero_simple_counter (&sm->total_users, 0);
2495   sm->total_sessions.name = "total-sessions";
2496   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2497   vlib_validate_simple_counter (&sm->total_sessions, 0);
2498   vlib_zero_simple_counter (&sm->total_sessions, 0);
2499
2500   /* Init IPFIX logging */
2501   snat_ipfix_logging_init (vm);
2502
2503   /* Init NAT64 */
2504   error = nat64_init (vm);
2505   if (error)
2506     return error;
2507
2508   dslite_init (vm);
2509
2510   nat66_init (vm);
2511
2512   ip4_table_bind_callback_t cbt4 = {
2513     .function = snat_ip4_table_bind,
2514   };
2515   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2516
2517   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2518                                         FIB_SOURCE_PRIORITY_HI,
2519                                         FIB_SOURCE_BH_SIMPLE);
2520   nat_fib_src_low = fib_source_allocate ("nat-low",
2521                                          FIB_SOURCE_PRIORITY_LOW,
2522                                          FIB_SOURCE_BH_SIMPLE);
2523
2524   return error;
2525 }
2526
2527 VLIB_INIT_FUNCTION (snat_init);
2528
2529 void
2530 snat_free_outside_address_and_port (snat_address_t * addresses,
2531                                     u32 thread_index, snat_session_key_t * k)
2532 {
2533   snat_address_t *a;
2534   u32 address_index;
2535   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2536
2537   for (address_index = 0; address_index < vec_len (addresses);
2538        address_index++)
2539     {
2540       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2541         break;
2542     }
2543
2544   ASSERT (address_index < vec_len (addresses));
2545
2546   a = addresses + address_index;
2547
2548   switch (k->protocol)
2549     {
2550 #define _(N, i, n, s) \
2551     case SNAT_PROTOCOL_##N: \
2552       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2553         port_host_byte_order) == 1); \
2554       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2555         port_host_byte_order, 0); \
2556       a->busy_##n##_ports--; \
2557       a->busy_##n##_ports_per_thread[thread_index]--; \
2558       break;
2559       foreach_snat_protocol
2560 #undef _
2561     default:
2562       nat_elog_info ("unknown protocol");
2563       return;
2564     }
2565 }
2566
2567 static int
2568 nat_set_outside_address_and_port (snat_address_t * addresses,
2569                                   u32 thread_index, snat_session_key_t * k)
2570 {
2571   snat_address_t *a = 0;
2572   u32 address_index;
2573   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2574
2575   for (address_index = 0; address_index < vec_len (addresses);
2576        address_index++)
2577     {
2578       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2579         continue;
2580
2581       a = addresses + address_index;
2582       switch (k->protocol)
2583         {
2584 #define _(N, j, n, s) \
2585         case SNAT_PROTOCOL_##N: \
2586           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2587             return VNET_API_ERROR_INSTANCE_IN_USE; \
2588           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2589           a->busy_##n##_ports_per_thread[thread_index]++; \
2590           a->busy_##n##_ports++; \
2591           return 0;
2592           foreach_snat_protocol
2593 #undef _
2594         default:
2595           nat_elog_info ("unknown protocol");
2596           return 1;
2597         }
2598     }
2599
2600   return VNET_API_ERROR_NO_SUCH_ENTRY;
2601 }
2602
2603 int
2604 snat_static_mapping_match (snat_main_t * sm,
2605                            snat_session_key_t match,
2606                            snat_session_key_t * mapping,
2607                            u8 by_external,
2608                            u8 * is_addr_only,
2609                            twice_nat_type_t * twice_nat,
2610                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2611                            u8 * is_identity_nat)
2612 {
2613   clib_bihash_kv_8_8_t kv, value;
2614   snat_static_mapping_t *m;
2615   snat_session_key_t m_key;
2616   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2617   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2618   u8 backend_index;
2619   nat44_lb_addr_port_t *local;
2620
2621   m_key.fib_index = match.fib_index;
2622   if (by_external)
2623     {
2624       mapping_hash = &sm->static_mapping_by_external;
2625       m_key.fib_index = 0;
2626     }
2627
2628   m_key.addr = match.addr;
2629   m_key.port = clib_net_to_host_u16 (match.port);
2630   m_key.protocol = match.protocol;
2631
2632   kv.key = m_key.as_u64;
2633
2634   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2635     {
2636       /* Try address only mapping */
2637       m_key.port = 0;
2638       m_key.protocol = 0;
2639       kv.key = m_key.as_u64;
2640       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2641         return 1;
2642     }
2643
2644   m = pool_elt_at_index (sm->static_mappings, value.value);
2645
2646   if (by_external)
2647     {
2648       if (is_lb_static_mapping (m))
2649         {
2650           if (PREDICT_FALSE (lb != 0))
2651             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2652           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2653                                                           match.addr,
2654                                                           match.protocol,
2655                                                           match.port,
2656                                                           &backend_index))
2657             {
2658               local = pool_elt_at_index (m->locals, backend_index);
2659               mapping->addr = local->addr;
2660               mapping->port = clib_host_to_net_u16 (local->port);
2661               mapping->fib_index = local->fib_index;
2662               goto end;
2663             }
2664           // pick locals matching this worker
2665           if (PREDICT_FALSE (sm->num_workers > 1))
2666             {
2667               u32 thread_index = vlib_get_thread_index ();
2668               /* *INDENT-OFF* */
2669               pool_foreach_index (i, m->locals,
2670               ({
2671                 local = pool_elt_at_index (m->locals, i);
2672
2673                 ip4_header_t ip = {
2674                   .src_address = local->addr,
2675                 };
2676
2677                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2678                     thread_index)
2679                   {
2680                     vec_add1 (tmp, i);
2681                   }
2682               }));
2683               /* *INDENT-ON* */
2684               ASSERT (vec_len (tmp) != 0);
2685             }
2686           else
2687             {
2688               /* *INDENT-OFF* */
2689               pool_foreach_index (i, m->locals,
2690               ({
2691                 vec_add1 (tmp, i);
2692               }));
2693               /* *INDENT-ON* */
2694             }
2695           hi = vec_len (tmp) - 1;
2696           local = pool_elt_at_index (m->locals, tmp[hi]);
2697           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2698           while (lo < hi)
2699             {
2700               mid = ((hi - lo) >> 1) + lo;
2701               local = pool_elt_at_index (m->locals, tmp[mid]);
2702               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2703             }
2704           local = pool_elt_at_index (m->locals, tmp[lo]);
2705           if (!(local->prefix >= rand))
2706             return 1;
2707           mapping->addr = local->addr;
2708           mapping->port = clib_host_to_net_u16 (local->port);
2709           mapping->fib_index = local->fib_index;
2710           if (m->affinity)
2711             {
2712               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2713                                                 match.protocol, match.port,
2714                                                 tmp[lo], m->affinity,
2715                                                 m->affinity_per_service_list_head_index))
2716                 nat_elog_info ("create affinity record failed");
2717             }
2718           vec_free (tmp);
2719         }
2720       else
2721         {
2722           if (PREDICT_FALSE (lb != 0))
2723             *lb = NO_LB_NAT;
2724           mapping->fib_index = m->fib_index;
2725           mapping->addr = m->local_addr;
2726           /* Address only mapping doesn't change port */
2727           mapping->port = is_addr_only_static_mapping (m) ? match.port
2728             : clib_host_to_net_u16 (m->local_port);
2729         }
2730       mapping->protocol = m->proto;
2731     }
2732   else
2733     {
2734       mapping->addr = m->external_addr;
2735       /* Address only mapping doesn't change port */
2736       mapping->port = is_addr_only_static_mapping (m) ? match.port
2737         : clib_host_to_net_u16 (m->external_port);
2738       mapping->fib_index = sm->outside_fib_index;
2739     }
2740
2741 end:
2742   if (PREDICT_FALSE (is_addr_only != 0))
2743     *is_addr_only = is_addr_only_static_mapping (m);
2744
2745   if (PREDICT_FALSE (twice_nat != 0))
2746     *twice_nat = m->twice_nat;
2747
2748   if (PREDICT_FALSE (is_identity_nat != 0))
2749     *is_identity_nat = is_identity_static_mapping (m);
2750
2751   return 0;
2752 }
2753
2754 static_always_inline u16
2755 snat_random_port (u16 min, u16 max)
2756 {
2757   snat_main_t *sm = &snat_main;
2758   return min + random_u32 (&sm->random_seed) /
2759     (random_u32_max () / (max - min + 1) + 1);
2760 }
2761
2762 int
2763 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2764                                      u32 fib_index,
2765                                      u32 thread_index,
2766                                      snat_session_key_t * k,
2767                                      u16 port_per_thread,
2768                                      u32 snat_thread_index)
2769 {
2770   snat_main_t *sm = &snat_main;
2771
2772   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2773                                   port_per_thread, snat_thread_index);
2774 }
2775
2776 static int
2777 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2778                                  u32 fib_index,
2779                                  u32 thread_index,
2780                                  snat_session_key_t * k,
2781                                  u16 port_per_thread, u32 snat_thread_index)
2782 {
2783   int i;
2784   snat_address_t *a, *ga = 0;
2785   u32 portnum;
2786
2787   for (i = 0; i < vec_len (addresses); i++)
2788     {
2789       a = addresses + i;
2790       switch (k->protocol)
2791         {
2792 #define _(N, j, n, s) \
2793         case SNAT_PROTOCOL_##N: \
2794           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2795             { \
2796               if (a->fib_index == fib_index) \
2797                 { \
2798                   while (1) \
2799                     { \
2800                       portnum = (port_per_thread * \
2801                         snat_thread_index) + \
2802                         snat_random_port(1, port_per_thread) + 1024; \
2803                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2804                         continue; \
2805                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2806                       a->busy_##n##_ports_per_thread[thread_index]++; \
2807                       a->busy_##n##_ports++; \
2808                       k->addr = a->addr; \
2809                       k->port = clib_host_to_net_u16(portnum); \
2810                       return 0; \
2811                     } \
2812                 } \
2813               else if (a->fib_index == ~0) \
2814                 { \
2815                   ga = a; \
2816                 } \
2817             } \
2818           break;
2819           foreach_snat_protocol
2820 #undef _
2821         default:
2822           nat_elog_info ("unknown protocol");
2823           return 1;
2824         }
2825
2826     }
2827
2828   if (ga)
2829     {
2830       a = ga;
2831       switch (k->protocol)
2832         {
2833 #define _(N, j, n, s) \
2834         case SNAT_PROTOCOL_##N: \
2835           while (1) \
2836             { \
2837               portnum = (port_per_thread * \
2838                 snat_thread_index) + \
2839                 snat_random_port(1, port_per_thread) + 1024; \
2840               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2841                 continue; \
2842               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2843               a->busy_##n##_ports_per_thread[thread_index]++; \
2844               a->busy_##n##_ports++; \
2845               k->addr = a->addr; \
2846               k->port = clib_host_to_net_u16(portnum); \
2847               return 0; \
2848             }
2849           break;
2850           foreach_snat_protocol
2851 #undef _
2852         default:
2853           nat_elog_info ("unknown protocol");
2854           return 1;
2855         }
2856     }
2857
2858   /* Totally out of translations to use... */
2859   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2860   return 1;
2861 }
2862
2863 static int
2864 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2865                               u32 fib_index,
2866                               u32 thread_index,
2867                               snat_session_key_t * k,
2868                               u16 port_per_thread, u32 snat_thread_index)
2869 {
2870   snat_main_t *sm = &snat_main;
2871   snat_address_t *a = addresses;
2872   u16 m, ports, portnum, A, j;
2873   m = 16 - (sm->psid_offset + sm->psid_length);
2874   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2875
2876   if (!vec_len (addresses))
2877     goto exhausted;
2878
2879   switch (k->protocol)
2880     {
2881 #define _(N, i, n, s) \
2882     case SNAT_PROTOCOL_##N: \
2883       if (a->busy_##n##_ports < ports) \
2884         { \
2885           while (1) \
2886             { \
2887               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2888               j = snat_random_port(0, pow2_mask(m)); \
2889               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2890               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2891                 continue; \
2892               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2893               a->busy_##n##_ports++; \
2894               k->addr = a->addr; \
2895               k->port = clib_host_to_net_u16 (portnum); \
2896               return 0; \
2897             } \
2898         } \
2899       break;
2900       foreach_snat_protocol
2901 #undef _
2902     default:
2903       nat_elog_info ("unknown protocol");
2904       return 1;
2905     }
2906
2907 exhausted:
2908   /* Totally out of translations to use... */
2909   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2910   return 1;
2911 }
2912
2913 static int
2914 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2915                                u32 fib_index,
2916                                u32 thread_index,
2917                                snat_session_key_t * k,
2918                                u16 port_per_thread, u32 snat_thread_index)
2919 {
2920   snat_main_t *sm = &snat_main;
2921   snat_address_t *a = addresses;
2922   u16 portnum, ports;
2923
2924   ports = sm->end_port - sm->start_port + 1;
2925
2926   if (!vec_len (addresses))
2927     goto exhausted;
2928
2929   switch (k->protocol)
2930     {
2931 #define _(N, i, n, s) \
2932     case SNAT_PROTOCOL_##N: \
2933       if (a->busy_##n##_ports < ports) \
2934         { \
2935           while (1) \
2936             { \
2937               portnum = snat_random_port(sm->start_port, sm->end_port); \
2938               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2939                 continue; \
2940               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2941               a->busy_##n##_ports++; \
2942               k->addr = a->addr; \
2943               k->port = clib_host_to_net_u16 (portnum); \
2944               return 0; \
2945             } \
2946         } \
2947       break;
2948       foreach_snat_protocol
2949 #undef _
2950     default:
2951       nat_elog_info ("unknown protocol");
2952       return 1;
2953     }
2954
2955 exhausted:
2956   /* Totally out of translations to use... */
2957   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2958   return 1;
2959 }
2960
2961 void
2962 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2963 {
2964   dpo_id_t dpo_v4 = DPO_INVALID;
2965   fib_prefix_t pfx = {
2966     .fp_proto = FIB_PROTOCOL_IP4,
2967     .fp_len = 32,
2968     .fp_addr.ip4.as_u32 = addr.as_u32,
2969   };
2970
2971   if (is_add)
2972     {
2973       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2974       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
2975                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2976       dpo_reset (&dpo_v4);
2977     }
2978   else
2979     {
2980       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
2981     }
2982 }
2983
2984 u8 *
2985 format_session_kvp (u8 * s, va_list * args)
2986 {
2987   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2988   snat_session_key_t k;
2989
2990   k.as_u64 = v->key;
2991
2992   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2993
2994   return s;
2995 }
2996
2997 u8 *
2998 format_static_mapping_kvp (u8 * s, va_list * args)
2999 {
3000   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3001   snat_session_key_t k;
3002
3003   k.as_u64 = v->key;
3004
3005   s = format (s, "%U static-mapping-index %llu",
3006               format_static_mapping_key, &k, v->value);
3007
3008   return s;
3009 }
3010
3011 u8 *
3012 format_user_kvp (u8 * s, va_list * args)
3013 {
3014   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3015   snat_user_key_t k;
3016
3017   k.as_u64 = v->key;
3018
3019   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3020               k.fib_index, v->value);
3021
3022   return s;
3023 }
3024
3025 u8 *
3026 format_ed_session_kvp (u8 * s, va_list * args)
3027 {
3028   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3029   nat_ed_ses_key_t k;
3030
3031   k.as_u64[0] = v->key[0];
3032   k.as_u64[1] = v->key[1];
3033
3034   s =
3035     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3036             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
3037             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
3038             format_ip_protocol, k.proto, k.fib_index, v->value);
3039
3040   return s;
3041 }
3042
3043 static u32
3044 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3045                            u8 is_output)
3046 {
3047   snat_main_t *sm = &snat_main;
3048   u32 next_worker_index = 0;
3049   u32 hash;
3050
3051   next_worker_index = sm->first_worker_index;
3052   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3053     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3054
3055   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3056     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3057   else
3058     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3059
3060   return next_worker_index;
3061 }
3062
3063 static u32
3064 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3065                            u32 rx_fib_index0, u8 is_output)
3066 {
3067   snat_main_t *sm = &snat_main;
3068   udp_header_t *udp;
3069   u16 port;
3070   snat_session_key_t m_key;
3071   clib_bihash_kv_8_8_t kv, value;
3072   snat_static_mapping_t *m;
3073   u32 proto;
3074   u32 next_worker_index = 0;
3075
3076   /* first try static mappings without port */
3077   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3078     {
3079       m_key.addr = ip0->dst_address;
3080       m_key.port = 0;
3081       m_key.protocol = 0;
3082       m_key.fib_index = rx_fib_index0;
3083       kv.key = m_key.as_u64;
3084       if (!clib_bihash_search_8_8
3085           (&sm->static_mapping_by_external, &kv, &value))
3086         {
3087           m = pool_elt_at_index (sm->static_mappings, value.value);
3088           return m->workers[0];
3089         }
3090     }
3091
3092   proto = ip_proto_to_snat_proto (ip0->protocol);
3093   udp = ip4_next_header (ip0);
3094   port = udp->dst_port;
3095
3096   /* unknown protocol */
3097   if (PREDICT_FALSE (proto == ~0))
3098     {
3099       /* use current thread */
3100       return vlib_get_thread_index ();
3101     }
3102
3103   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3104     {
3105       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3106       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3107       if (!icmp_type_is_error_message
3108           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3109         port = vnet_buffer (b)->ip.reass.l4_src_port;
3110       else
3111         {
3112           /* if error message, then it's not fragmented and we can access it */
3113           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3114           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3115           void *l4_header = ip4_next_header (inner_ip);
3116           switch (proto)
3117             {
3118             case SNAT_PROTOCOL_ICMP:
3119               icmp = (icmp46_header_t *) l4_header;
3120               echo = (icmp_echo_header_t *) (icmp + 1);
3121               port = echo->identifier;
3122               break;
3123             case SNAT_PROTOCOL_UDP:
3124             case SNAT_PROTOCOL_TCP:
3125               port = ((tcp_udp_header_t *) l4_header)->src_port;
3126               break;
3127             default:
3128               return vlib_get_thread_index ();
3129             }
3130         }
3131     }
3132
3133   /* try static mappings with port */
3134   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3135     {
3136       m_key.addr = ip0->dst_address;
3137       m_key.port = clib_net_to_host_u16 (port);
3138       m_key.protocol = proto;
3139       m_key.fib_index = rx_fib_index0;
3140       kv.key = m_key.as_u64;
3141       if (!clib_bihash_search_8_8
3142           (&sm->static_mapping_by_external, &kv, &value))
3143         {
3144           m = pool_elt_at_index (sm->static_mappings, value.value);
3145           return m->workers[0];
3146         }
3147     }
3148
3149   /* worker by outside port */
3150   next_worker_index = sm->first_worker_index;
3151   next_worker_index +=
3152     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3153   return next_worker_index;
3154 }
3155
3156 static u32
3157 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3158                                u8 is_output)
3159 {
3160   snat_main_t *sm = &snat_main;
3161   u32 next_worker_index = sm->first_worker_index;
3162   u32 hash;
3163
3164   clib_bihash_kv_16_8_t kv16, value16;
3165   snat_main_per_thread_data_t *tsm;
3166   udp_header_t *udp;
3167
3168   if (PREDICT_FALSE (is_output))
3169     {
3170       u32 fib_index = sm->outside_fib_index;
3171       nat_outside_fib_t *outside_fib;
3172       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3173       fib_prefix_t pfx = {
3174         .fp_proto = FIB_PROTOCOL_IP4,
3175         .fp_len = 32,
3176         .fp_addr = {
3177                     .ip4.as_u32 = ip->dst_address.as_u32,
3178                     }
3179         ,
3180       };
3181
3182       udp = ip4_next_header (ip);
3183
3184       switch (vec_len (sm->outside_fibs))
3185         {
3186         case 0:
3187           fib_index = sm->outside_fib_index;
3188           break;
3189         case 1:
3190           fib_index = sm->outside_fibs[0].fib_index;
3191           break;
3192         default:
3193             /* *INDENT-OFF* */
3194             vec_foreach (outside_fib, sm->outside_fibs)
3195               {
3196                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3197                 if (FIB_NODE_INDEX_INVALID != fei)
3198                   {
3199                     if (fib_entry_get_resolving_interface (fei) != ~0)
3200                       {
3201                         fib_index = outside_fib->fib_index;
3202                         break;
3203                       }
3204                   }
3205               }
3206             /* *INDENT-ON* */
3207           break;
3208         }
3209
3210       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3211                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3212
3213       /* *INDENT-OFF* */
3214       vec_foreach (tsm, sm->per_thread_data)
3215         {
3216           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3217                                                       &kv16, &value16)))
3218             {
3219               next_worker_index += tsm->thread_index;
3220
3221               nat_elog_debug_handoff (
3222                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3223                 next_worker_index, fib_index,
3224                 clib_net_to_host_u32 (ip->src_address.as_u32),
3225                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3226
3227               return next_worker_index;
3228             }
3229         }
3230       /* *INDENT-ON* */
3231     }
3232
3233   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3234     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3235
3236   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3237     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3238   else
3239     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3240
3241   if (PREDICT_TRUE (!is_output))
3242     {
3243       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3244                               next_worker_index, rx_fib_index,
3245                               clib_net_to_host_u32 (ip->src_address.as_u32),
3246                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3247     }
3248   else
3249     {
3250       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3251                               next_worker_index, rx_fib_index,
3252                               clib_net_to_host_u32 (ip->src_address.as_u32),
3253                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3254     }
3255
3256   return next_worker_index;
3257 }
3258
3259 static u32
3260 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3261                                u32 rx_fib_index, u8 is_output)
3262 {
3263   snat_main_t *sm = &snat_main;
3264   clib_bihash_kv_8_8_t kv, value;
3265   clib_bihash_kv_16_8_t kv16, value16;
3266   snat_main_per_thread_data_t *tsm;
3267
3268   u32 proto, next_worker_index = 0;
3269   udp_header_t *udp;
3270   u16 port;
3271   snat_static_mapping_t *m;
3272   u32 hash;
3273
3274   proto = ip_proto_to_snat_proto (ip->protocol);
3275
3276   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3277     {
3278       udp = ip4_next_header (ip);
3279
3280       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3281                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3282
3283       /* *INDENT-OFF* */
3284       vec_foreach (tsm, sm->per_thread_data)
3285         {
3286           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3287                                                       &kv16, &value16)))
3288             {
3289               next_worker_index = sm->first_worker_index + tsm->thread_index;
3290               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3291                           next_worker_index, rx_fib_index,
3292                           clib_net_to_host_u32 (ip->src_address.as_u32),
3293                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3294               return next_worker_index;
3295             }
3296           }
3297         /* *INDENT-ON* */
3298     }
3299   else if (proto == SNAT_PROTOCOL_ICMP)
3300     {
3301       nat_ed_ses_key_t key;
3302
3303       if (!get_icmp_o2i_ed_key (b, ip, &key))
3304         {
3305
3306           key.fib_index = rx_fib_index;
3307           kv16.key[0] = key.as_u64[0];
3308           kv16.key[1] = key.as_u64[1];
3309
3310           /* *INDENT-OFF* */
3311           vec_foreach (tsm, sm->per_thread_data)
3312             {
3313               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3314                                                           &kv16, &value16)))
3315                 {
3316                   next_worker_index = sm->first_worker_index +
3317                                       tsm->thread_index;
3318                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3319                               next_worker_index, rx_fib_index,
3320                               clib_net_to_host_u32 (ip->src_address.as_u32),
3321                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3322                   return next_worker_index;
3323                 }
3324             }
3325           /* *INDENT-ON* */
3326         }
3327     }
3328
3329   /* first try static mappings without port */
3330   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3331     {
3332       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3333       if (!clib_bihash_search_8_8
3334           (&sm->static_mapping_by_external, &kv, &value))
3335         {
3336           m = pool_elt_at_index (sm->static_mappings, value.value);
3337           next_worker_index = m->workers[0];
3338           goto done;
3339         }
3340     }
3341
3342   /* unknown protocol */
3343   if (PREDICT_FALSE (proto == ~0))
3344     {
3345       /* use current thread */
3346       next_worker_index = vlib_get_thread_index ();
3347       goto done;
3348     }
3349
3350   udp = ip4_next_header (ip);
3351   port = udp->dst_port;
3352
3353   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3354     {
3355       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3356       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3357       if (!icmp_type_is_error_message
3358           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3359         port = vnet_buffer (b)->ip.reass.l4_src_port;
3360       else
3361         {
3362           /* if error message, then it's not fragmented and we can access it */
3363           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3364           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3365           void *l4_header = ip4_next_header (inner_ip);
3366           switch (proto)
3367             {
3368             case SNAT_PROTOCOL_ICMP:
3369               icmp = (icmp46_header_t *) l4_header;
3370               echo = (icmp_echo_header_t *) (icmp + 1);
3371               port = echo->identifier;
3372               break;
3373             case SNAT_PROTOCOL_UDP:
3374             case SNAT_PROTOCOL_TCP:
3375               port = ((tcp_udp_header_t *) l4_header)->src_port;
3376               break;
3377             default:
3378               next_worker_index = vlib_get_thread_index ();
3379               goto done;
3380             }
3381         }
3382     }
3383
3384   /* try static mappings with port */
3385   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3386     {
3387       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3388                   clib_net_to_host_u16 (port));
3389       if (!clib_bihash_search_8_8
3390           (&sm->static_mapping_by_external, &kv, &value))
3391         {
3392           m = pool_elt_at_index (sm->static_mappings, value.value);
3393           if (!is_lb_static_mapping (m))
3394             {
3395               next_worker_index = m->workers[0];
3396               goto done;
3397             }
3398
3399           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3400             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3401
3402           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3403             next_worker_index =
3404               m->workers[hash & (_vec_len (m->workers) - 1)];
3405           else
3406             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3407           goto done;
3408         }
3409     }
3410
3411   /* worker by outside port */
3412   next_worker_index = sm->first_worker_index;
3413   next_worker_index +=
3414     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3415
3416 done:
3417   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3418                           clib_net_to_host_u32 (ip->src_address.as_u32),
3419                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3420   return next_worker_index;
3421 }
3422
3423 void
3424 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3425                 ip4_address_t * out_addr, u16 out_port,
3426                 ip4_address_t * eh_addr, u16 eh_port,
3427                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3428                 u32 fib_index, u16 flags, u32 thread_index)
3429 {
3430   snat_main_t *sm = &snat_main;
3431   snat_session_key_t key;
3432   snat_user_t *u;
3433   snat_session_t *s;
3434   clib_bihash_kv_8_8_t kv;
3435   f64 now = vlib_time_now (sm->vlib_main);
3436   nat_outside_fib_t *outside_fib;
3437   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3438   snat_main_per_thread_data_t *tsm;
3439   fib_prefix_t pfx = {
3440     .fp_proto = FIB_PROTOCOL_IP4,
3441     .fp_len = 32,
3442     .fp_addr = {
3443                 .ip4.as_u32 = eh_addr->as_u32,
3444                 },
3445   };
3446
3447   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3448
3449   key.addr.as_u32 = out_addr->as_u32;
3450   key.port = out_port;
3451   key.protocol = proto;
3452
3453   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3454     {
3455       if (nat_set_outside_address_and_port
3456           (sm->addresses, thread_index, &key))
3457         return;
3458     }
3459
3460   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3461   if (!u)
3462     return;
3463
3464   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3465   if (!s)
3466     return;
3467
3468   s->last_heard = now;
3469   s->flags = flags;
3470   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3471   s->ext_host_port = eh_port;
3472   user_session_increment (sm, u, snat_is_session_static (s));
3473   switch (vec_len (sm->outside_fibs))
3474     {
3475     case 0:
3476       key.fib_index = sm->outside_fib_index;
3477       break;
3478     case 1:
3479       key.fib_index = sm->outside_fibs[0].fib_index;
3480       break;
3481     default:
3482       /* *INDENT-OFF* */
3483       vec_foreach (outside_fib, sm->outside_fibs)
3484         {
3485           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3486           if (FIB_NODE_INDEX_INVALID != fei)
3487             {
3488               if (fib_entry_get_resolving_interface (fei) != ~0)
3489                 {
3490                   key.fib_index = outside_fib->fib_index;
3491                   break;
3492                 }
3493             }
3494         }
3495       /* *INDENT-ON* */
3496       break;
3497     }
3498   s->out2in = key;
3499   kv.key = key.as_u64;
3500   kv.value = s - tsm->sessions;
3501   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3502     nat_elog_warn ("out2in key add failed");
3503
3504   key.addr.as_u32 = in_addr->as_u32;
3505   key.port = in_port;
3506   key.fib_index = fib_index;
3507   s->in2out = key;
3508   kv.key = key.as_u64;
3509   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3510     nat_elog_warn ("in2out key add failed");
3511 }
3512
3513 void
3514 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3515                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3516                 u32 ti)
3517 {
3518   snat_main_t *sm = &snat_main;
3519   snat_session_key_t key;
3520   clib_bihash_kv_8_8_t kv, value;
3521   u32 thread_index;
3522   snat_session_t *s;
3523   snat_main_per_thread_data_t *tsm;
3524
3525   if (sm->num_workers > 1)
3526     thread_index =
3527       sm->first_worker_index +
3528       (sm->workers[(clib_net_to_host_u16 (out_port) -
3529                     1024) / sm->port_per_thread]);
3530   else
3531     thread_index = sm->num_workers;
3532   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3533
3534   key.addr.as_u32 = out_addr->as_u32;
3535   key.port = out_port;
3536   key.protocol = proto;
3537   key.fib_index = fib_index;
3538   kv.key = key.as_u64;
3539   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3540     return;
3541
3542   s = pool_elt_at_index (tsm->sessions, value.value);
3543   nat_free_session_data (sm, s, thread_index, 1);
3544   nat44_delete_session (sm, s, thread_index);
3545 }
3546
3547 void
3548 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3549                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3550                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3551 {
3552   snat_main_t *sm = &snat_main;
3553   snat_session_key_t key;
3554   clib_bihash_kv_8_8_t kv, value;
3555   snat_session_t *s;
3556   snat_main_per_thread_data_t *tsm;
3557
3558   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3559
3560   key.addr.as_u32 = out_addr->as_u32;
3561   key.port = out_port;
3562   key.protocol = proto;
3563   key.fib_index = fib_index;
3564   kv.key = key.as_u64;
3565   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3566     return;
3567
3568   s = pool_elt_at_index (tsm->sessions, value.value);
3569   s->total_pkts = total_pkts;
3570   s->total_bytes = total_bytes;
3571 }
3572
3573 void
3574 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3575                    ip4_address_t * out_addr, u16 out_port,
3576                    ip4_address_t * eh_addr, u16 eh_port,
3577                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3578                    u32 fib_index, u16 flags, u32 thread_index)
3579 {
3580   snat_main_t *sm = &snat_main;
3581   snat_session_key_t key;
3582   snat_user_t *u;
3583   snat_session_t *s;
3584   clib_bihash_kv_16_8_t kv;
3585   f64 now = vlib_time_now (sm->vlib_main);
3586   nat_outside_fib_t *outside_fib;
3587   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3588   snat_main_per_thread_data_t *tsm;
3589   fib_prefix_t pfx = {
3590     .fp_proto = FIB_PROTOCOL_IP4,
3591     .fp_len = 32,
3592     .fp_addr = {
3593                 .ip4.as_u32 = eh_addr->as_u32,
3594                 },
3595   };
3596
3597   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3598
3599   key.addr.as_u32 = out_addr->as_u32;
3600   key.port = out_port;
3601   key.protocol = proto;
3602
3603   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3604     {
3605       if (nat_set_outside_address_and_port
3606           (sm->addresses, thread_index, &key))
3607         return;
3608     }
3609
3610   key.addr.as_u32 = ehn_addr->as_u32;
3611   key.port = ehn_port;
3612   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3613     {
3614       if (nat_set_outside_address_and_port
3615           (sm->twice_nat_addresses, thread_index, &key))
3616         return;
3617     }
3618
3619   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3620   if (!u)
3621     return;
3622
3623   s = nat_ed_session_alloc (sm, u, thread_index, now);
3624   if (!s)
3625     return;
3626
3627   s->last_heard = now;
3628   s->flags = flags;
3629   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3630   s->ext_host_nat_port = s->ext_host_port = eh_port;
3631   if (is_twice_nat_session (s))
3632     {
3633       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3634       s->ext_host_nat_port = ehn_port;
3635     }
3636   user_session_increment (sm, u, snat_is_session_static (s));
3637   switch (vec_len (sm->outside_fibs))
3638     {
3639     case 0:
3640       key.fib_index = sm->outside_fib_index;
3641       break;
3642     case 1:
3643       key.fib_index = sm->outside_fibs[0].fib_index;
3644       break;
3645     default:
3646       /* *INDENT-OFF* */
3647       vec_foreach (outside_fib, sm->outside_fibs)
3648         {
3649           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3650           if (FIB_NODE_INDEX_INVALID != fei)
3651             {
3652               if (fib_entry_get_resolving_interface (fei) != ~0)
3653                 {
3654                   key.fib_index = outside_fib->fib_index;
3655                   break;
3656                 }
3657             }
3658         }
3659       /* *INDENT-ON* */
3660       break;
3661     }
3662   key.addr.as_u32 = out_addr->as_u32;
3663   key.port = out_port;
3664   s->out2in = key;
3665   kv.value = s - tsm->sessions;
3666
3667   key.addr.as_u32 = in_addr->as_u32;
3668   key.port = in_port;
3669   key.fib_index = fib_index;
3670   s->in2out = key;
3671
3672   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3673               snat_proto_to_ip_proto (proto), fib_index, in_port,
3674               s->ext_host_nat_port);
3675   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3676     nat_elog_warn ("in2out key add failed");
3677
3678   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3679               s->out2in.fib_index, out_port, eh_port);
3680   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3681     nat_elog_warn ("out2in key add failed");
3682 }
3683
3684 void
3685 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3686                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3687                    u32 fib_index, u32 ti)
3688 {
3689   snat_main_t *sm = &snat_main;
3690   nat_ed_ses_key_t key;
3691   clib_bihash_kv_16_8_t kv, value;
3692   u32 thread_index;
3693   snat_session_t *s;
3694   snat_main_per_thread_data_t *tsm;
3695
3696   if (sm->num_workers > 1)
3697     thread_index =
3698       sm->first_worker_index +
3699       (sm->workers[(clib_net_to_host_u16 (out_port) -
3700                     1024) / sm->port_per_thread]);
3701   else
3702     thread_index = sm->num_workers;
3703   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3704
3705   key.l_addr.as_u32 = out_addr->as_u32;
3706   key.l_port = out_port;
3707   key.r_addr.as_u32 = eh_addr->as_u32;
3708   key.r_port = eh_port;
3709   key.proto = proto;
3710   key.fib_index = fib_index;
3711   kv.key[0] = key.as_u64[0];
3712   kv.key[1] = key.as_u64[1];
3713   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3714     return;
3715
3716   s = pool_elt_at_index (tsm->sessions, value.value);
3717   nat_free_session_data (sm, s, thread_index, 1);
3718   nat44_delete_session (sm, s, thread_index);
3719 }
3720
3721 void
3722 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3723                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3724                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3725                    u32 thread_index)
3726 {
3727   snat_main_t *sm = &snat_main;
3728   nat_ed_ses_key_t key;
3729   clib_bihash_kv_16_8_t kv, value;
3730   snat_session_t *s;
3731   snat_main_per_thread_data_t *tsm;
3732
3733   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3734
3735   key.l_addr.as_u32 = out_addr->as_u32;
3736   key.l_port = out_port;
3737   key.r_addr.as_u32 = eh_addr->as_u32;
3738   key.r_port = eh_port;
3739   key.proto = proto;
3740   key.fib_index = fib_index;
3741   kv.key[0] = key.as_u64[0];
3742   kv.key[1] = key.as_u64[1];
3743   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3744     return;
3745
3746   s = pool_elt_at_index (tsm->sessions, value.value);
3747   s->total_pkts = total_pkts;
3748   s->total_bytes = total_bytes;
3749 }
3750
3751 static clib_error_t *
3752 snat_config (vlib_main_t * vm, unformat_input_t * input)
3753 {
3754   snat_main_t *sm = &snat_main;
3755   nat66_main_t *nm = &nat66_main;
3756   dslite_main_t *dm = &dslite_main;
3757   snat_main_per_thread_data_t *tsm;
3758
3759   u32 static_mapping_buckets = 1024;
3760   u32 static_mapping_memory_size = 64 << 20;
3761
3762   u32 nat64_bib_buckets = 1024;
3763   u32 nat64_bib_memory_size = 128 << 20;
3764
3765   u32 nat64_st_buckets = 2048;
3766   u32 nat64_st_memory_size = 256 << 20;
3767
3768   u32 user_buckets = 128;
3769   u32 user_memory_size = 64 << 20;
3770   u32 translation_buckets = 1024;
3771   u32 translation_memory_size = 128 << 20;
3772
3773   u32 max_translations_per_user = ~0;
3774
3775   u32 outside_vrf_id = 0;
3776   u32 outside_ip6_vrf_id = 0;
3777   u32 inside_vrf_id = 0;
3778   u8 static_mapping_only = 0;
3779   u8 static_mapping_connection_tracking = 0;
3780
3781   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3782   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3783
3784   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3785   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3786
3787   sm->deterministic = 0;
3788   sm->out2in_dpo = 0;
3789   sm->endpoint_dependent = 0;
3790
3791   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3792     {
3793       if (unformat
3794           (input, "translation hash buckets %d", &translation_buckets))
3795         ;
3796       else if (unformat (input, "udp timeout %d", &udp_timeout))
3797         ;
3798       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3799         ;
3800       else if (unformat (input, "tcp transitory timeout %d",
3801                          &tcp_transitory_timeout));
3802       else if (unformat (input, "tcp established timeout %d",
3803                          &tcp_established_timeout));
3804       else if (unformat (input, "translation hash memory %d",
3805                          &translation_memory_size));
3806       else if (unformat (input, "user hash buckets %d", &user_buckets))
3807         ;
3808       else if (unformat (input, "user hash memory %d", &user_memory_size))
3809         ;
3810       else if (unformat (input, "max translations per user %d",
3811                          &max_translations_per_user))
3812         ;
3813       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3814         ;
3815       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3816         ;
3817       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3818         ;
3819       else if (unformat (input, "static mapping only"))
3820         {
3821           static_mapping_only = 1;
3822           if (unformat (input, "connection tracking"))
3823             static_mapping_connection_tracking = 1;
3824         }
3825       else if (unformat (input, "deterministic"))
3826         sm->deterministic = 1;
3827       else if (unformat (input, "nat64 bib hash buckets %d",
3828                          &nat64_bib_buckets))
3829         ;
3830       else if (unformat (input, "nat64 bib hash memory %d",
3831                          &nat64_bib_memory_size))
3832         ;
3833       else
3834         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3835         ;
3836       else if (unformat (input, "nat64 st hash memory %d",
3837                          &nat64_st_memory_size))
3838         ;
3839       else if (unformat (input, "out2in dpo"))
3840         sm->out2in_dpo = 1;
3841       else if (unformat (input, "dslite ce"))
3842         dslite_set_ce (dm, 1);
3843       else if (unformat (input, "endpoint-dependent"))
3844         sm->endpoint_dependent = 1;
3845       else
3846         return clib_error_return (0, "unknown input '%U'",
3847                                   format_unformat_error, input);
3848     }
3849
3850   if (sm->deterministic && sm->endpoint_dependent)
3851     return clib_error_return (0,
3852                               "deterministic and endpoint-dependent modes are mutually exclusive");
3853
3854   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3855     return clib_error_return (0,
3856                               "static mapping only mode available only for simple nat");
3857
3858   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3859     return clib_error_return (0,
3860                               "out2in dpo mode available only for simple nat");
3861
3862   /* optionally configurable timeouts for testing purposes */
3863   sm->udp_timeout = udp_timeout;
3864   sm->icmp_timeout = icmp_timeout;
3865   sm->tcp_transitory_timeout = tcp_transitory_timeout;
3866   sm->tcp_established_timeout = tcp_established_timeout;
3867
3868   sm->user_buckets = user_buckets;
3869   sm->user_memory_size = user_memory_size;
3870
3871   sm->translation_buckets = translation_buckets;
3872   sm->translation_memory_size = translation_memory_size;
3873
3874   /* do not exceed load factor 10 */
3875   sm->max_translations = 10 * translation_buckets;
3876   sm->max_translations_per_user = max_translations_per_user == ~0 ?
3877     sm->max_translations : max_translations_per_user;
3878
3879   sm->outside_vrf_id = outside_vrf_id;
3880   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3881                                                              outside_vrf_id,
3882                                                              nat_fib_src_hi);
3883   nm->outside_vrf_id = outside_ip6_vrf_id;
3884   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3885                                                              outside_ip6_vrf_id,
3886                                                              nat_fib_src_hi);
3887   sm->inside_vrf_id = inside_vrf_id;
3888   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3889                                                             inside_vrf_id,
3890                                                             nat_fib_src_hi);
3891   sm->static_mapping_only = static_mapping_only;
3892   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3893
3894   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3895                   nat64_st_memory_size);
3896
3897   if (sm->deterministic)
3898     {
3899       sm->in2out_node_index = snat_det_in2out_node.index;
3900       sm->in2out_output_node_index = ~0;
3901       sm->out2in_node_index = snat_det_out2in_node.index;
3902       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3903       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3904     }
3905   else
3906     {
3907       if (sm->endpoint_dependent)
3908         {
3909           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
3910           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3911
3912           sm->handoff_out2in_index = nat_pre_out2in_node.index;
3913           sm->handoff_in2out_index = nat_pre_in2out_node.index;
3914           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
3915
3916           sm->in2out_node_index = nat44_ed_in2out_node.index;
3917           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3918           sm->out2in_node_index = nat44_ed_out2in_node.index;
3919
3920           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3921           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3922           nat_affinity_init (vm);
3923           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
3924                        nat_ha_sref_ed_cb);
3925         }
3926       else
3927         {
3928           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3929           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3930
3931           sm->handoff_out2in_index = snat_in2out_node.index;
3932           sm->handoff_in2out_index = snat_out2in_node.index;
3933           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
3934
3935           sm->in2out_node_index = snat_in2out_node.index;
3936           sm->in2out_output_node_index = snat_in2out_output_node.index;
3937           sm->out2in_node_index = snat_out2in_node.index;
3938           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3939           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3940           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
3941         }
3942       if (!static_mapping_only ||
3943           (static_mapping_only && static_mapping_connection_tracking))
3944         {
3945           /* *INDENT-OFF* */
3946           vec_foreach (tsm, sm->per_thread_data)
3947             {
3948               if (sm->endpoint_dependent)
3949                 {
3950                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3951                                          translation_buckets,
3952                                          translation_memory_size);
3953                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3954                                                       format_ed_session_kvp);
3955
3956                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3957                                          translation_buckets,
3958                                          translation_memory_size);
3959                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3960                                                       format_ed_session_kvp);
3961                 }
3962               else
3963                 {
3964                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3965                                         translation_buckets,
3966                                         translation_memory_size);
3967                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3968                                                      format_session_kvp);
3969
3970                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3971                                         translation_buckets,
3972                                         translation_memory_size);
3973                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3974                                                      format_session_kvp);
3975                 }
3976
3977               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3978                                     user_memory_size);
3979               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3980                                                  format_user_kvp);
3981             }
3982           /* *INDENT-ON* */
3983
3984         }
3985       else
3986         {
3987           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3988           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3989         }
3990       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3991                             "static_mapping_by_local", static_mapping_buckets,
3992                             static_mapping_memory_size);
3993       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3994                                          format_static_mapping_kvp);
3995
3996       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3997                             "static_mapping_by_external",
3998                             static_mapping_buckets,
3999                             static_mapping_memory_size);
4000       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4001                                          format_static_mapping_kvp);
4002     }
4003
4004   return 0;
4005 }
4006
4007 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4008
4009 static void
4010 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4011                                  uword opaque,
4012                                  u32 sw_if_index,
4013                                  ip4_address_t * address,
4014                                  u32 address_length,
4015                                  u32 if_address_index, u32 is_delete)
4016 {
4017   snat_main_t *sm = &snat_main;
4018   snat_static_map_resolve_t *rp;
4019   snat_static_mapping_t *m;
4020   snat_session_key_t m_key;
4021   clib_bihash_kv_8_8_t kv, value;
4022   int i, rv;
4023   ip4_address_t l_addr;
4024
4025   for (i = 0; i < vec_len (sm->to_resolve); i++)
4026     {
4027       rp = sm->to_resolve + i;
4028       if (rp->addr_only == 0)
4029         continue;
4030       if (rp->sw_if_index == sw_if_index)
4031         goto match;
4032     }
4033
4034   return;
4035
4036 match:
4037   m_key.addr.as_u32 = address->as_u32;
4038   m_key.port = rp->addr_only ? 0 : rp->e_port;
4039   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4040   m_key.fib_index = sm->outside_fib_index;
4041   kv.key = m_key.as_u64;
4042   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4043     m = 0;
4044   else
4045     m = pool_elt_at_index (sm->static_mappings, value.value);
4046
4047   if (!is_delete)
4048     {
4049       /* Don't trip over lease renewal, static config */
4050       if (m)
4051         return;
4052     }
4053   else
4054     {
4055       if (!m)
4056         return;
4057     }
4058
4059   /* Indetity mapping? */
4060   if (rp->l_addr.as_u32 == 0)
4061     l_addr.as_u32 = address[0].as_u32;
4062   else
4063     l_addr.as_u32 = rp->l_addr.as_u32;
4064   /* Add the static mapping */
4065   rv = snat_add_static_mapping (l_addr,
4066                                 address[0],
4067                                 rp->l_port,
4068                                 rp->e_port,
4069                                 rp->vrf_id,
4070                                 rp->addr_only, ~0 /* sw_if_index */ ,
4071                                 rp->proto, !is_delete, rp->twice_nat,
4072                                 rp->out2in_only, rp->tag, rp->identity_nat);
4073   if (rv)
4074     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4075 }
4076
4077 static void
4078 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4079                                        uword opaque,
4080                                        u32 sw_if_index,
4081                                        ip4_address_t * address,
4082                                        u32 address_length,
4083                                        u32 if_address_index, u32 is_delete)
4084 {
4085   snat_main_t *sm = &snat_main;
4086   snat_static_map_resolve_t *rp;
4087   ip4_address_t l_addr;
4088   int i, j;
4089   int rv;
4090   u8 twice_nat = 0;
4091   snat_address_t *addresses = sm->addresses;
4092
4093   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4094     {
4095       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4096         goto match;
4097     }
4098
4099   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4100     {
4101       twice_nat = 1;
4102       addresses = sm->twice_nat_addresses;
4103       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4104         goto match;
4105     }
4106
4107   return;
4108
4109 match:
4110   if (!is_delete)
4111     {
4112       /* Don't trip over lease renewal, static config */
4113       for (j = 0; j < vec_len (addresses); j++)
4114         if (addresses[j].addr.as_u32 == address->as_u32)
4115           return;
4116
4117       (void) snat_add_address (sm, address, ~0, twice_nat);
4118       /* Scan static map resolution vector */
4119       for (j = 0; j < vec_len (sm->to_resolve); j++)
4120         {
4121           rp = sm->to_resolve + j;
4122           if (rp->addr_only)
4123             continue;
4124           /* On this interface? */
4125           if (rp->sw_if_index == sw_if_index)
4126             {
4127               /* Indetity mapping? */
4128               if (rp->l_addr.as_u32 == 0)
4129                 l_addr.as_u32 = address[0].as_u32;
4130               else
4131                 l_addr.as_u32 = rp->l_addr.as_u32;
4132               /* Add the static mapping */
4133               rv = snat_add_static_mapping (l_addr,
4134                                             address[0],
4135                                             rp->l_port,
4136                                             rp->e_port,
4137                                             rp->vrf_id,
4138                                             rp->addr_only,
4139                                             ~0 /* sw_if_index */ ,
4140                                             rp->proto,
4141                                             rp->is_add, rp->twice_nat,
4142                                             rp->out2in_only, rp->tag,
4143                                             rp->identity_nat);
4144               if (rv)
4145                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4146                                     "i4", rv);
4147             }
4148         }
4149       return;
4150     }
4151   else
4152     {
4153       (void) snat_del_address (sm, address[0], 1, twice_nat);
4154       return;
4155     }
4156 }
4157
4158
4159 int
4160 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4161                             u8 twice_nat)
4162 {
4163   ip4_main_t *ip4_main = sm->ip4_main;
4164   ip4_address_t *first_int_addr;
4165   snat_static_map_resolve_t *rp;
4166   u32 *indices_to_delete = 0;
4167   int i, j;
4168   u32 *auto_add_sw_if_indices =
4169     twice_nat ? sm->
4170     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4171
4172   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4173     );
4174
4175   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4176     {
4177       if (auto_add_sw_if_indices[i] == sw_if_index)
4178         {
4179           if (is_del)
4180             {
4181               /* if have address remove it */
4182               if (first_int_addr)
4183                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4184               else
4185                 {
4186                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4187                     {
4188                       rp = sm->to_resolve + j;
4189                       if (rp->sw_if_index == sw_if_index)
4190                         vec_add1 (indices_to_delete, j);
4191                     }
4192                   if (vec_len (indices_to_delete))
4193                     {
4194                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4195                         vec_del1 (sm->to_resolve, j);
4196                       vec_free (indices_to_delete);
4197                     }
4198                 }
4199               if (twice_nat)
4200                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4201               else
4202                 vec_del1 (sm->auto_add_sw_if_indices, i);
4203             }
4204           else
4205             return VNET_API_ERROR_VALUE_EXIST;
4206
4207           return 0;
4208         }
4209     }
4210
4211   if (is_del)
4212     return VNET_API_ERROR_NO_SUCH_ENTRY;
4213
4214   /* add to the auto-address list */
4215   if (twice_nat)
4216     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4217   else
4218     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4219
4220   /* If the address is already bound - or static - add it now */
4221   if (first_int_addr)
4222     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4223
4224   return 0;
4225 }
4226
4227 int
4228 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4229                    snat_protocol_t proto, u32 vrf_id, int is_in)
4230 {
4231   snat_main_per_thread_data_t *tsm;
4232   clib_bihash_kv_8_8_t kv, value;
4233   ip4_header_t ip;
4234   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4235   snat_session_key_t key;
4236   snat_session_t *s;
4237   clib_bihash_8_8_t *t;
4238
4239   if (sm->endpoint_dependent)
4240     return VNET_API_ERROR_UNSUPPORTED;
4241
4242   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4243   if (sm->num_workers > 1)
4244     tsm =
4245       vec_elt_at_index (sm->per_thread_data,
4246                         sm->worker_in2out_cb (&ip, fib_index, 0));
4247   else
4248     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4249
4250   key.addr.as_u32 = addr->as_u32;
4251   key.port = clib_host_to_net_u16 (port);
4252   key.protocol = proto;
4253   key.fib_index = fib_index;
4254   kv.key = key.as_u64;
4255   t = is_in ? &tsm->in2out : &tsm->out2in;
4256   if (!clib_bihash_search_8_8 (t, &kv, &value))
4257     {
4258       if (pool_is_free_index (tsm->sessions, value.value))
4259         return VNET_API_ERROR_UNSPECIFIED;
4260
4261       s = pool_elt_at_index (tsm->sessions, value.value);
4262       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4263       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4264       return 0;
4265     }
4266
4267   return VNET_API_ERROR_NO_SUCH_ENTRY;
4268 }
4269
4270 int
4271 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4272                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4273                       u32 vrf_id, int is_in)
4274 {
4275   ip4_header_t ip;
4276   clib_bihash_16_8_t *t;
4277   nat_ed_ses_key_t key;
4278   clib_bihash_kv_16_8_t kv, value;
4279   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4280   snat_session_t *s;
4281   snat_main_per_thread_data_t *tsm;
4282
4283   if (!sm->endpoint_dependent)
4284     return VNET_API_ERROR_FEATURE_DISABLED;
4285
4286   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4287   if (sm->num_workers > 1)
4288     tsm =
4289       vec_elt_at_index (sm->per_thread_data,
4290                         sm->worker_in2out_cb (&ip, fib_index, 0));
4291   else
4292     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4293
4294   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4295   key.l_addr.as_u32 = addr->as_u32;
4296   key.r_addr.as_u32 = eh_addr->as_u32;
4297   key.l_port = clib_host_to_net_u16 (port);
4298   key.r_port = clib_host_to_net_u16 (eh_port);
4299   key.proto = proto;
4300   key.fib_index = fib_index;
4301   kv.key[0] = key.as_u64[0];
4302   kv.key[1] = key.as_u64[1];
4303   if (clib_bihash_search_16_8 (t, &kv, &value))
4304     return VNET_API_ERROR_NO_SUCH_ENTRY;
4305
4306   if (pool_is_free_index (tsm->sessions, value.value))
4307     return VNET_API_ERROR_UNSPECIFIED;
4308   s = pool_elt_at_index (tsm->sessions, value.value);
4309   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4310   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4311   return 0;
4312 }
4313
4314 void
4315 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4316 {
4317   snat_main_t *sm = &snat_main;
4318
4319   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4320   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4321   sm->psid = psid;
4322   sm->psid_offset = psid_offset;
4323   sm->psid_length = psid_length;
4324 }
4325
4326 void
4327 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4328 {
4329   snat_main_t *sm = &snat_main;
4330
4331   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4332   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4333   sm->start_port = start_port;
4334   sm->end_port = end_port;
4335 }
4336
4337 void
4338 nat_set_alloc_addr_and_port_default (void)
4339 {
4340   snat_main_t *sm = &snat_main;
4341
4342   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4343   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4344 }
4345
4346 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4347                                  vlib_node_runtime_t * node,
4348                                  vlib_frame_t * frame)
4349 {
4350   return 0;
4351 }
4352
4353 /* *INDENT-OFF* */
4354 VLIB_REGISTER_NODE (nat_default_node) = {
4355   .name = "nat-default",
4356   .vector_size = sizeof (u32),
4357   .format_trace = 0,
4358   .type = VLIB_NODE_TYPE_INTERNAL,
4359   .n_errors = 0,
4360   .n_next_nodes = NAT_N_NEXT,
4361   .next_nodes = {
4362     [NAT_NEXT_DROP] = "error-drop",
4363     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4364     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4365     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4366     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4367     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4368     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4369     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4370     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4371   },
4372 };
4373 /* *INDENT-ON* */
4374
4375 /*
4376  * fd.io coding-style-patch-verification: ON
4377  *
4378  * Local Variables:
4379  * eval: (c-set-style "gnu")
4380  * End:
4381  */