src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35
  36 #include <vpp/app/version.h>
  37
  38 snat_main_t snat_main;
  39
  40 /* *INDENT-OFF* */
  41
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat44-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  47 };
  48 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat44-out2in",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-dhcp-client-detect"),
  53 };
  54 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-classify",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-in2out",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-out2in",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-det-classify",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  74 };
  75 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-ed-in2out",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-ed-out2in",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  84                                "ip4-dhcp-client-detect"),
  85 };
  86 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-ed-classify",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-in2out-worker-handoff",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  95 };
  96 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-out2in-worker-handoff",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-handoff-classify",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-in2out-fast",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 111 };
 112 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-out2in-fast",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 116                                "ip4-dhcp-client-detect"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-hairpin-dst",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ed-hairpin-dst",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 127 };
 128
 129 /* Hook up output features */
 130 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 131   .arc_name = "ip4-output",
 132   .node_name = "nat44-in2out-output",
 133   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 136   .arc_name = "ip4-output",
 137   .node_name = "nat44-in2out-output-worker-handoff",
 138   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 141   .arc_name = "ip4-output",
 142   .node_name = "nat44-hairpin-src",
 143   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 144 };
 145 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-ed-in2out-output",
 148   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 149 };
 150 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-ed-hairpin-src",
 153   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 154 };
 155
 156 /* Hook up ip4-local features */
 157 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 158 {
 159   .arc_name = "ip4-local",
 160   .node_name = "nat44-hairpinning",
 161   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 164 {
 165   .arc_name = "ip4-local",
 166   .node_name = "nat44-ed-hairpinning",
 167   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 168 };
 169
 170
 171 VLIB_PLUGIN_REGISTER () = {
 172     .version = VPP_BUILD_VER,
 173     .description = "Network Address Translation",
 174 };
 175 /* *INDENT-ON* */
 176
 177 void
 178 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 179 {
 180   snat_session_key_t key;
 181   clib_bihash_kv_8_8_t kv;
 182   nat_ed_ses_key_t ed_key;
 183   clib_bihash_kv_16_8_t ed_kv;
 184   snat_main_per_thread_data_t *tsm =
 185     vec_elt_at_index (sm->per_thread_data, thread_index);
 186
 187   if (is_fwd_bypass_session (s))
 188     {
 189       ed_key.l_addr = s->in2out.addr;
 190       ed_key.r_addr = s->ext_host_addr;
 191       ed_key.l_port = s->in2out.port;
 192       ed_key.r_port = s->ext_host_port;
 193       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 194       ed_key.fib_index = 0;
 195       ed_kv.key[0] = ed_key.as_u64[0];
 196       ed_kv.key[1] = ed_key.as_u64[1];
 197       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 198         nat_log_warn ("in2out_ed key del failed");
 199       return;
 200     }
 201
 202   /* session lookup tables */
 203   if (is_ed_session (s))
 204     {
 205       if (is_affinity_sessions (s))
 206         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 207                              s->in2out.protocol, s->out2in.port);
 208       ed_key.l_addr = s->out2in.addr;
 209       ed_key.r_addr = s->ext_host_addr;
 210       ed_key.fib_index = s->out2in.fib_index;
 211       if (snat_is_unk_proto_session (s))
 212         {
 213           ed_key.proto = s->in2out.port;
 214           ed_key.r_port = 0;
 215           ed_key.l_port = 0;
 216         }
 217       else
 218         {
 219           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 220           ed_key.l_port = s->out2in.port;
 221           ed_key.r_port = s->ext_host_port;
 222         }
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 226         nat_log_warn ("out2in_ed key del failed");
 227       ed_key.l_addr = s->in2out.addr;
 228       ed_key.fib_index = s->in2out.fib_index;
 229       if (!snat_is_unk_proto_session (s))
 230         ed_key.l_port = s->in2out.port;
 231       if (is_twice_nat_session (s))
 232         {
 233           ed_key.r_addr = s->ext_host_nat_addr;
 234           ed_key.r_port = s->ext_host_nat_port;
 235         }
 236       ed_kv.key[0] = ed_key.as_u64[0];
 237       ed_kv.key[1] = ed_key.as_u64[1];
 238       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 239         nat_log_warn ("in2out_ed key del failed");
 240
 241       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 242                              &s->in2out.addr, s->in2out.port,
 243                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 244                              &s->out2in.addr, s->out2in.port,
 245                              &s->ext_host_addr, s->ext_host_port,
 246                              s->in2out.protocol, is_twice_nat_session (s));
 247     }
 248   else
 249     {
 250       kv.key = s->in2out.as_u64;
 251       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 252         nat_log_warn ("in2out key del failed");
 253       kv.key = s->out2in.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 255         nat_log_warn ("out2in key del failed");
 256
 257       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 258                                &s->in2out.addr, s->in2out.port,
 259                                &s->out2in.addr, s->out2in.port,
 260                                s->in2out.protocol);
 261     }
 262
 263   if (snat_is_unk_proto_session (s))
 264     return;
 265
 266   /* log NAT event */
 267   snat_ipfix_logging_nat44_ses_delete (thread_index,
 268                                        s->in2out.addr.as_u32,
 269                                        s->out2in.addr.as_u32,
 270                                        s->in2out.protocol,
 271                                        s->in2out.port,
 272                                        s->out2in.port, s->in2out.fib_index);
 273
 274   /* Twice NAT address and port for external host */
 275   if (is_twice_nat_session (s))
 276     {
 277       key.protocol = s->in2out.protocol;
 278       key.port = s->ext_host_nat_port;
 279       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 280       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 281                                           thread_index, &key);
 282     }
 283
 284   if (snat_is_session_static (s))
 285     return;
 286
 287   snat_free_outside_address_and_port (sm->addresses, thread_index,
 288                                       &s->out2in);
 289 }
 290
 291 snat_user_t *
 292 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 293                         u32 thread_index)
 294 {
 295   snat_user_t *u = 0;
 296   snat_user_key_t user_key;
 297   clib_bihash_kv_8_8_t kv, value;
 298   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 299   dlist_elt_t *per_user_list_head_elt;
 300
 301   user_key.addr.as_u32 = addr->as_u32;
 302   user_key.fib_index = fib_index;
 303   kv.key = user_key.as_u64;
 304
 305   /* Ever heard of the "user" = src ip4 address before? */
 306   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 307     {
 308       /* no, make a new one */
 309       pool_get (tsm->users, u);
 310       clib_memset (u, 0, sizeof (*u));
 311       u->addr.as_u32 = addr->as_u32;
 312       u->fib_index = fib_index;
 313
 314       pool_get (tsm->list_pool, per_user_list_head_elt);
 315
 316       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 317         tsm->list_pool;
 318
 319       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 320
 321       kv.value = u - tsm->users;
 322
 323       /* add user */
 324       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 325         nat_log_warn ("user_hash keay add failed");
 326
 327       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 328                                pool_elts (tsm->users));
 329     }
 330   else
 331     {
 332       u = pool_elt_at_index (tsm->users, value.value);
 333     }
 334
 335   return u;
 336 }
 337
 338 snat_session_t *
 339 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 340                               u32 thread_index)
 341 {
 342   snat_session_t *s;
 343   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 344   u32 oldest_per_user_translation_list_index, session_index;
 345   dlist_elt_t *oldest_per_user_translation_list_elt;
 346   dlist_elt_t *per_user_translation_list_elt;
 347
 348   /* Over quota? Recycle the least recently used translation */
 349   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 350     {
 351       oldest_per_user_translation_list_index =
 352         clib_dlist_remove_head (tsm->list_pool,
 353                                 u->sessions_per_user_list_head_index);
 354
 355       ASSERT (oldest_per_user_translation_list_index != ~0);
 356
 357       /* Add it back to the end of the LRU list */
 358       clib_dlist_addtail (tsm->list_pool,
 359                           u->sessions_per_user_list_head_index,
 360                           oldest_per_user_translation_list_index);
 361       /* Get the list element */
 362       oldest_per_user_translation_list_elt =
 363         pool_elt_at_index (tsm->list_pool,
 364                            oldest_per_user_translation_list_index);
 365
 366       /* Get the session index from the list element */
 367       session_index = oldest_per_user_translation_list_elt->value;
 368
 369       /* Get the session */
 370       s = pool_elt_at_index (tsm->sessions, session_index);
 371       nat_free_session_data (sm, s, thread_index);
 372       if (snat_is_session_static (s))
 373         u->nstaticsessions--;
 374       else
 375         u->nsessions--;
 376       s->flags = 0;
 377       s->total_bytes = 0;
 378       s->total_pkts = 0;
 379       s->state = 0;
 380       s->ext_host_addr.as_u32 = 0;
 381       s->ext_host_port = 0;
 382       s->ext_host_nat_addr.as_u32 = 0;
 383       s->ext_host_nat_port = 0;
 384     }
 385   else
 386     {
 387       pool_get (tsm->sessions, s);
 388       clib_memset (s, 0, sizeof (*s));
 389
 390       /* Create list elts */
 391       pool_get (tsm->list_pool, per_user_translation_list_elt);
 392       clib_dlist_init (tsm->list_pool,
 393                        per_user_translation_list_elt - tsm->list_pool);
 394
 395       per_user_translation_list_elt->value = s - tsm->sessions;
 396       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 397       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 398
 399       clib_dlist_addtail (tsm->list_pool,
 400                           s->per_user_list_head_index,
 401                           per_user_translation_list_elt - tsm->list_pool);
 402
 403       s->user_index = u - tsm->users;
 404       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 405                                pool_elts (tsm->sessions));
 406     }
 407
 408   return s;
 409 }
 410
 411 snat_session_t *
 412 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 413                       f64 now)
 414 {
 415   snat_session_t *s;
 416   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 417   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 418   u32 oldest_index;
 419   u64 sess_timeout_time;
 420
 421   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 422     goto alloc_new;
 423
 424   oldest_index =
 425     clib_dlist_remove_head (tsm->list_pool,
 426                             u->sessions_per_user_list_head_index);
 427   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 428   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 429   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 430   if (now >= sess_timeout_time)
 431     {
 432       clib_dlist_addtail (tsm->list_pool,
 433                           u->sessions_per_user_list_head_index, oldest_index);
 434       nat_free_session_data (sm, s, thread_index);
 435       if (snat_is_session_static (s))
 436         u->nstaticsessions--;
 437       else
 438         u->nsessions--;
 439       s->flags = 0;
 440       s->total_bytes = 0;
 441       s->total_pkts = 0;
 442       s->state = 0;
 443       s->ext_host_addr.as_u32 = 0;
 444       s->ext_host_port = 0;
 445       s->ext_host_nat_addr.as_u32 = 0;
 446       s->ext_host_nat_port = 0;
 447     }
 448   else
 449     {
 450       clib_dlist_addhead (tsm->list_pool,
 451                           u->sessions_per_user_list_head_index, oldest_index);
 452       if ((u->nsessions + u->nstaticsessions) >=
 453           sm->max_translations_per_user)
 454         {
 455           nat_log_warn ("max translations per user %U", format_ip4_address,
 456                         &u->addr);
 457           snat_ipfix_logging_max_entries_per_user
 458             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 459           return 0;
 460         }
 461       else
 462         {
 463         alloc_new:
 464           pool_get (tsm->sessions, s);
 465           clib_memset (s, 0, sizeof (*s));
 466
 467           /* Create list elts */
 468           pool_get (tsm->list_pool, per_user_translation_list_elt);
 469           clib_dlist_init (tsm->list_pool,
 470                            per_user_translation_list_elt - tsm->list_pool);
 471
 472           per_user_translation_list_elt->value = s - tsm->sessions;
 473           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 474           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 475
 476           clib_dlist_addtail (tsm->list_pool,
 477                               s->per_user_list_head_index,
 478                               per_user_translation_list_elt - tsm->list_pool);
 479         }
 480
 481       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 482                                pool_elts (tsm->sessions));
 483     }
 484
 485   return s;
 486 }
 487
 488 void
 489 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 490                           int is_add)
 491 {
 492   fib_prefix_t prefix = {
 493     .fp_len = p_len,
 494     .fp_proto = FIB_PROTOCOL_IP4,
 495     .fp_addr = {
 496                 .ip4.as_u32 = addr->as_u32,
 497                 },
 498   };
 499   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 500
 501   if (is_add)
 502     fib_table_entry_update_one_path (fib_index,
 503                                      &prefix,
 504                                      FIB_SOURCE_PLUGIN_LOW,
 505                                      (FIB_ENTRY_FLAG_CONNECTED |
 506                                       FIB_ENTRY_FLAG_LOCAL |
 507                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 508                                      DPO_PROTO_IP4,
 509                                      NULL,
 510                                      sw_if_index,
 511                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 512   else
 513     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 514 }
 515
 516 int
 517 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 518                   u8 twice_nat)
 519 {
 520   snat_address_t *ap;
 521   snat_interface_t *i;
 522   vlib_thread_main_t *tm = vlib_get_thread_main ();
 523
 524   if (twice_nat && !sm->endpoint_dependent)
 525     return VNET_API_ERROR_FEATURE_DISABLED;
 526
 527   /* Check if address already exists */
 528   /* *INDENT-OFF* */
 529   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 530     {
 531       if (ap->addr.as_u32 == addr->as_u32)
 532         return VNET_API_ERROR_VALUE_EXIST;
 533     }
 534   /* *INDENT-ON* */
 535
 536   if (twice_nat)
 537     vec_add2 (sm->twice_nat_addresses, ap, 1);
 538   else
 539     vec_add2 (sm->addresses, ap, 1);
 540
 541   ap->addr = *addr;
 542   if (vrf_id != ~0)
 543     ap->fib_index =
 544       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 545                                          FIB_SOURCE_PLUGIN_LOW);
 546   else
 547     ap->fib_index = ~0;
 548 #define _(N, i, n, s) \
 549   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 550   ap->busy_##n##_ports = 0; \
 551   ap->busy_##n##_ports_per_thread = 0;\
 552   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 553   foreach_snat_protocol
 554 #undef _
 555     if (twice_nat)
 556     return 0;
 557
 558   /* Add external address to FIB */
 559   /* *INDENT-OFF* */
 560   pool_foreach (i, sm->interfaces,
 561   ({
 562     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 563       continue;
 564
 565     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 566     break;
 567   }));
 568   pool_foreach (i, sm->output_feature_interfaces,
 569   ({
 570     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 571       continue;
 572
 573     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 574     break;
 575   }));
 576   /* *INDENT-ON* */
 577
 578   return 0;
 579 }
 580
 581 static int
 582 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 583 {
 584   snat_static_mapping_t *m;
 585   /* *INDENT-OFF* */
 586   pool_foreach (m, sm->static_mappings,
 587   ({
 588       if (is_addr_only_static_mapping (m) ||
 589           is_out2in_only_static_mapping (m) ||
 590           is_identity_static_mapping (m))
 591         continue;
 592       if (m->external_addr.as_u32 == addr.as_u32)
 593         return 1;
 594   }));
 595   /* *INDENT-ON* */
 596
 597   return 0;
 598 }
 599
 600 void
 601 increment_v4_address (ip4_address_t * a)
 602 {
 603   u32 v;
 604
 605   v = clib_net_to_host_u32 (a->as_u32) + 1;
 606   a->as_u32 = clib_host_to_net_u32 (v);
 607 }
 608
 609 static void
 610 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 611                                        ip4_address_t l_addr,
 612                                        u16 l_port,
 613                                        u32 sw_if_index,
 614                                        u16 e_port,
 615                                        u32 vrf_id,
 616                                        snat_protocol_t proto,
 617                                        int addr_only, int is_add, u8 * tag,
 618                                        int twice_nat, int out2in_only,
 619                                        int identity_nat)
 620 {
 621   snat_static_map_resolve_t *rp;
 622
 623   vec_add2 (sm->to_resolve, rp, 1);
 624   rp->l_addr.as_u32 = l_addr.as_u32;
 625   rp->l_port = l_port;
 626   rp->sw_if_index = sw_if_index;
 627   rp->e_port = e_port;
 628   rp->vrf_id = vrf_id;
 629   rp->proto = proto;
 630   rp->addr_only = addr_only;
 631   rp->is_add = is_add;
 632   rp->twice_nat = twice_nat;
 633   rp->out2in_only = out2in_only;
 634   rp->identity_nat = identity_nat;
 635   rp->tag = vec_dup (tag);
 636 }
 637
 638 static u32
 639 get_thread_idx_by_port (u16 e_port)
 640 {
 641   snat_main_t *sm = &snat_main;
 642   u32 thread_idx = sm->num_workers;
 643   if (sm->num_workers > 1)
 644     {
 645       thread_idx =
 646         sm->first_worker_index +
 647         sm->workers[(e_port - 1024) / sm->port_per_thread];
 648     }
 649   return thread_idx;
 650 }
 651
 652 int
 653 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 654                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 655                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 656                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 657                          u8 identity_nat)
 658 {
 659   snat_main_t *sm = &snat_main;
 660   snat_static_mapping_t *m;
 661   snat_session_key_t m_key;
 662   clib_bihash_kv_8_8_t kv, value;
 663   snat_address_t *a = 0;
 664   u32 fib_index = ~0;
 665   snat_interface_t *interface;
 666   int i;
 667   snat_main_per_thread_data_t *tsm;
 668   snat_user_key_t u_key;
 669   snat_user_t *u;
 670   dlist_elt_t *head, *elt;
 671   u32 elt_index, head_index;
 672   u32 ses_index;
 673   u64 user_index;
 674   snat_session_t *s;
 675   snat_static_map_resolve_t *rp, *rp_match = 0;
 676   nat44_lb_addr_port_t *local;
 677   u32 find = ~0;
 678
 679   if (!sm->endpoint_dependent)
 680     {
 681       if (twice_nat || out2in_only)
 682         return VNET_API_ERROR_FEATURE_DISABLED;
 683     }
 684
 685   /* If the external address is a specific interface address */
 686   if (sw_if_index != ~0)
 687     {
 688       ip4_address_t *first_int_addr;
 689
 690       for (i = 0; i < vec_len (sm->to_resolve); i++)
 691         {
 692           rp = sm->to_resolve + i;
 693           if (rp->sw_if_index != sw_if_index ||
 694               rp->l_addr.as_u32 != l_addr.as_u32 ||
 695               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 696             continue;
 697
 698           if (!addr_only)
 699             {
 700               if (rp->l_port != l_port || rp->e_port != e_port
 701                   || rp->proto != proto)
 702                 continue;
 703             }
 704
 705           rp_match = rp;
 706           break;
 707         }
 708
 709       /* Might be already set... */
 710       first_int_addr = ip4_interface_first_address
 711         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 712
 713       if (is_add)
 714         {
 715           if (rp_match)
 716             return VNET_API_ERROR_VALUE_EXIST;
 717
 718           snat_add_static_mapping_when_resolved
 719             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 720              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 721
 722           /* DHCP resolution required? */
 723           if (first_int_addr == 0)
 724             {
 725               return 0;
 726             }
 727           else
 728             {
 729               e_addr.as_u32 = first_int_addr->as_u32;
 730               /* Identity mapping? */
 731               if (l_addr.as_u32 == 0)
 732                 l_addr.as_u32 = e_addr.as_u32;
 733             }
 734         }
 735       else
 736         {
 737           if (!rp_match)
 738             return VNET_API_ERROR_NO_SUCH_ENTRY;
 739
 740           vec_del1 (sm->to_resolve, i);
 741
 742           if (first_int_addr)
 743             {
 744               e_addr.as_u32 = first_int_addr->as_u32;
 745               /* Identity mapping? */
 746               if (l_addr.as_u32 == 0)
 747                 l_addr.as_u32 = e_addr.as_u32;
 748             }
 749           else
 750             return 0;
 751         }
 752     }
 753
 754   m_key.addr = e_addr;
 755   m_key.port = addr_only ? 0 : e_port;
 756   m_key.protocol = addr_only ? 0 : proto;
 757   m_key.fib_index = 0;
 758   kv.key = m_key.as_u64;
 759   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 760     m = 0;
 761   else
 762     m = pool_elt_at_index (sm->static_mappings, value.value);
 763
 764   if (is_add)
 765     {
 766       if (m)
 767         {
 768           if (is_identity_static_mapping (m))
 769             {
 770               /* *INDENT-OFF* */
 771               pool_foreach (local, m->locals,
 772               ({
 773                 if (local->vrf_id == vrf_id)
 774                   return VNET_API_ERROR_VALUE_EXIST;
 775               }));
 776               /* *INDENT-ON* */
 777               pool_get (m->locals, local);
 778               local->vrf_id = vrf_id;
 779               local->fib_index =
 780                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 781                                                    FIB_SOURCE_PLUGIN_LOW);
 782               m_key.addr = m->local_addr;
 783               m_key.port = m->local_port;
 784               m_key.protocol = m->proto;
 785               m_key.fib_index = local->fib_index;
 786               kv.key = m_key.as_u64;
 787               kv.value = m - sm->static_mappings;
 788               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 789               return 0;
 790             }
 791           else
 792             return VNET_API_ERROR_VALUE_EXIST;
 793         }
 794
 795       if (twice_nat && addr_only)
 796         return VNET_API_ERROR_UNSUPPORTED;
 797
 798       /* Convert VRF id to FIB index */
 799       if (vrf_id != ~0)
 800         fib_index =
 801           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 802                                              FIB_SOURCE_PLUGIN_LOW);
 803       /* If not specified use inside VRF id from SNAT plugin startup config */
 804       else
 805         {
 806           fib_index = sm->inside_fib_index;
 807           vrf_id = sm->inside_vrf_id;
 808         }
 809
 810       if (!(out2in_only || identity_nat))
 811         {
 812           m_key.addr = l_addr;
 813           m_key.port = addr_only ? 0 : l_port;
 814           m_key.protocol = addr_only ? 0 : proto;
 815           m_key.fib_index = fib_index;
 816           kv.key = m_key.as_u64;
 817           if (!clib_bihash_search_8_8
 818               (&sm->static_mapping_by_local, &kv, &value))
 819             return VNET_API_ERROR_VALUE_EXIST;
 820         }
 821
 822       /* Find external address in allocated addresses and reserve port for
 823          address and port pair mapping when dynamic translations enabled */
 824       if (!(addr_only || sm->static_mapping_only || out2in_only))
 825         {
 826           for (i = 0; i < vec_len (sm->addresses); i++)
 827             {
 828               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 829                 {
 830                   a = sm->addresses + i;
 831                   /* External port must be unused */
 832                   switch (proto)
 833                     {
 834 #define _(N, j, n, s) \
 835                     case SNAT_PROTOCOL_##N: \
 836                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 837                         return VNET_API_ERROR_INVALID_VALUE; \
 838                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 839                       if (e_port > 1024) \
 840                         { \
 841                           a->busy_##n##_ports++; \
 842                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 843                         } \
 844                       break;
 845                       foreach_snat_protocol
 846 #undef _
 847                     default:
 848                       nat_log_info ("unknown protocol");
 849                       return VNET_API_ERROR_INVALID_VALUE_2;
 850                     }
 851                   break;
 852                 }
 853             }
 854           /* External address must be allocated */
 855           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 856             {
 857               if (sw_if_index != ~0)
 858                 {
 859                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 860                     {
 861                       rp = sm->to_resolve + i;
 862                       if (rp->addr_only)
 863                         continue;
 864                       if (rp->sw_if_index != sw_if_index &&
 865                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 866                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 867                           rp->e_port != e_port && rp->proto != proto)
 868                         continue;
 869
 870                       vec_del1 (sm->to_resolve, i);
 871                       break;
 872                     }
 873                 }
 874               return VNET_API_ERROR_NO_SUCH_ENTRY;
 875             }
 876         }
 877
 878       pool_get (sm->static_mappings, m);
 879       clib_memset (m, 0, sizeof (*m));
 880       m->tag = vec_dup (tag);
 881       m->local_addr = l_addr;
 882       m->external_addr = e_addr;
 883       m->twice_nat = twice_nat;
 884       if (out2in_only)
 885         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 886       if (addr_only)
 887         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 888       if (identity_nat)
 889         {
 890           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 891           pool_get (m->locals, local);
 892           local->vrf_id = vrf_id;
 893           local->fib_index = fib_index;
 894         }
 895       else
 896         {
 897           m->vrf_id = vrf_id;
 898           m->fib_index = fib_index;
 899         }
 900       if (!addr_only)
 901         {
 902           m->local_port = l_port;
 903           m->external_port = e_port;
 904           m->proto = proto;
 905         }
 906
 907       if (sm->num_workers > 1)
 908         {
 909           ip4_header_t ip = {
 910             .src_address = m->local_addr,
 911           };
 912           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 913           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 914         }
 915       else
 916         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 917
 918       m_key.addr = m->local_addr;
 919       m_key.port = m->local_port;
 920       m_key.protocol = m->proto;
 921       m_key.fib_index = fib_index;
 922       kv.key = m_key.as_u64;
 923       kv.value = m - sm->static_mappings;
 924       if (!out2in_only)
 925         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 926
 927       m_key.addr = m->external_addr;
 928       m_key.port = m->external_port;
 929       m_key.fib_index = 0;
 930       kv.key = m_key.as_u64;
 931       kv.value = m - sm->static_mappings;
 932       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 933
 934       /* Delete dynamic sessions matching local address (+ local port) */
 935       if (!(sm->static_mapping_only))
 936         {
 937           u_key.addr = m->local_addr;
 938           u_key.fib_index = m->fib_index;
 939           kv.key = u_key.as_u64;
 940           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 941             {
 942               user_index = value.value;
 943               u = pool_elt_at_index (tsm->users, user_index);
 944               if (u->nsessions)
 945                 {
 946                   head_index = u->sessions_per_user_list_head_index;
 947                   head = pool_elt_at_index (tsm->list_pool, head_index);
 948                   elt_index = head->next;
 949                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 950                   ses_index = elt->value;
 951                   while (ses_index != ~0)
 952                     {
 953                       s = pool_elt_at_index (tsm->sessions, ses_index);
 954                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 955                       ses_index = elt->value;
 956
 957                       if (snat_is_session_static (s))
 958                         continue;
 959
 960                       if (!addr_only
 961                           && (clib_net_to_host_u16 (s->in2out.port) !=
 962                               m->local_port))
 963                         continue;
 964
 965                       nat_free_session_data (sm, s,
 966                                              tsm - sm->per_thread_data);
 967                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 968
 969                       if (!addr_only && !sm->endpoint_dependent)
 970                         break;
 971                     }
 972                 }
 973             }
 974         }
 975     }
 976   else
 977     {
 978       if (!m)
 979         {
 980           if (sw_if_index != ~0)
 981             return 0;
 982           else
 983             return VNET_API_ERROR_NO_SUCH_ENTRY;
 984         }
 985
 986       if (identity_nat)
 987         {
 988           if (vrf_id == ~0)
 989             vrf_id = sm->inside_vrf_id;
 990
 991           /* *INDENT-OFF* */
 992           pool_foreach (local, m->locals,
 993           ({
 994             if (local->vrf_id == vrf_id)
 995               find = local - m->locals;
 996           }));
 997           /* *INDENT-ON* */
 998           if (find == ~0)
 999             return VNET_API_ERROR_NO_SUCH_ENTRY;
1000
1001           local = pool_elt_at_index (m->locals, find);
1002           fib_index = local->fib_index;
1003           pool_put (m->locals, local);
1004         }
1005       else
1006         fib_index = m->fib_index;
1007
1008       /* Free external address port */
1009       if (!(addr_only || sm->static_mapping_only || out2in_only))
1010         {
1011           for (i = 0; i < vec_len (sm->addresses); i++)
1012             {
1013               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1014                 {
1015                   a = sm->addresses + i;
1016                   switch (proto)
1017                     {
1018 #define _(N, j, n, s) \
1019                     case SNAT_PROTOCOL_##N: \
1020                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1021                       if (e_port > 1024) \
1022                         { \
1023                           a->busy_##n##_ports--; \
1024                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1025                         } \
1026                       break;
1027                       foreach_snat_protocol
1028 #undef _
1029                     default:
1030                       nat_log_info ("unknown protocol");
1031                       return VNET_API_ERROR_INVALID_VALUE_2;
1032                     }
1033                   break;
1034                 }
1035             }
1036         }
1037
1038       if (sm->num_workers > 1)
1039         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1040       else
1041         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1042
1043       m_key.addr = m->local_addr;
1044       m_key.port = m->local_port;
1045       m_key.protocol = m->proto;
1046       m_key.fib_index = fib_index;
1047       kv.key = m_key.as_u64;
1048       if (!out2in_only)
1049         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1050
1051       /* Delete session(s) for static mapping if exist */
1052       if (!(sm->static_mapping_only) ||
1053           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1054         {
1055           u_key.addr = m->local_addr;
1056           u_key.fib_index = fib_index;
1057           kv.key = u_key.as_u64;
1058           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1059             {
1060               user_index = value.value;
1061               u = pool_elt_at_index (tsm->users, user_index);
1062               if (u->nstaticsessions)
1063                 {
1064                   head_index = u->sessions_per_user_list_head_index;
1065                   head = pool_elt_at_index (tsm->list_pool, head_index);
1066                   elt_index = head->next;
1067                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1068                   ses_index = elt->value;
1069                   while (ses_index != ~0)
1070                     {
1071                       s = pool_elt_at_index (tsm->sessions, ses_index);
1072                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1073                       ses_index = elt->value;
1074
1075                       if (!addr_only)
1076                         {
1077                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1078                               (clib_net_to_host_u16 (s->out2in.port) !=
1079                                e_port))
1080                             continue;
1081                         }
1082
1083                       if (is_lb_session (s))
1084                         continue;
1085
1086                       if (!snat_is_session_static (s))
1087                         continue;
1088
1089                       nat_free_session_data (sm, s,
1090                                              tsm - sm->per_thread_data);
1091                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1092
1093                       if (!addr_only && !sm->endpoint_dependent)
1094                         break;
1095                     }
1096                 }
1097             }
1098         }
1099
1100       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1101       if (pool_elts (m->locals))
1102         return 0;
1103
1104       m_key.addr = m->external_addr;
1105       m_key.port = m->external_port;
1106       m_key.fib_index = 0;
1107       kv.key = m_key.as_u64;
1108       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1109
1110       vec_free (m->tag);
1111       vec_free (m->workers);
1112       /* Delete static mapping from pool */
1113       pool_put (sm->static_mappings, m);
1114     }
1115
1116   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1117     return 0;
1118
1119   /* Add/delete external address to FIB */
1120   /* *INDENT-OFF* */
1121   pool_foreach (interface, sm->interfaces,
1122   ({
1123     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1124       continue;
1125
1126     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1127     break;
1128   }));
1129   pool_foreach (interface, sm->output_feature_interfaces,
1130   ({
1131     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1132       continue;
1133
1134     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1135     break;
1136   }));
1137   /* *INDENT-ON* */
1138
1139   return 0;
1140 }
1141
1142 int
1143 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1144                                  snat_protocol_t proto,
1145                                  nat44_lb_addr_port_t * locals, u8 is_add,
1146                                  twice_nat_type_t twice_nat, u8 out2in_only,
1147                                  u8 * tag, u32 affinity)
1148 {
1149   snat_main_t *sm = &snat_main;
1150   snat_static_mapping_t *m;
1151   snat_session_key_t m_key;
1152   clib_bihash_kv_8_8_t kv, value;
1153   snat_address_t *a = 0;
1154   int i;
1155   nat44_lb_addr_port_t *local;
1156   u32 elt_index, head_index, ses_index;
1157   snat_main_per_thread_data_t *tsm;
1158   snat_user_key_t u_key;
1159   snat_user_t *u;
1160   snat_session_t *s;
1161   dlist_elt_t *head, *elt;
1162   uword *bitmap = 0;
1163
1164   if (!sm->endpoint_dependent)
1165     return VNET_API_ERROR_FEATURE_DISABLED;
1166
1167   m_key.addr = e_addr;
1168   m_key.port = e_port;
1169   m_key.protocol = proto;
1170   m_key.fib_index = 0;
1171   kv.key = m_key.as_u64;
1172   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1173     m = 0;
1174   else
1175     m = pool_elt_at_index (sm->static_mappings, value.value);
1176
1177   if (is_add)
1178     {
1179       if (m)
1180         return VNET_API_ERROR_VALUE_EXIST;
1181
1182       if (vec_len (locals) < 2)
1183         return VNET_API_ERROR_INVALID_VALUE;
1184
1185       /* Find external address in allocated addresses and reserve port for
1186          address and port pair mapping when dynamic translations enabled */
1187       if (!(sm->static_mapping_only || out2in_only))
1188         {
1189           for (i = 0; i < vec_len (sm->addresses); i++)
1190             {
1191               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1192                 {
1193                   a = sm->addresses + i;
1194                   /* External port must be unused */
1195                   switch (proto)
1196                     {
1197 #define _(N, j, n, s) \
1198                     case SNAT_PROTOCOL_##N: \
1199                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1200                         return VNET_API_ERROR_INVALID_VALUE; \
1201                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1202                       if (e_port > 1024) \
1203                         { \
1204                           a->busy_##n##_ports++; \
1205                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1206                         } \
1207                       break;
1208                       foreach_snat_protocol
1209 #undef _
1210                     default:
1211                       nat_log_info ("unknown protocol");
1212                       return VNET_API_ERROR_INVALID_VALUE_2;
1213                     }
1214                   break;
1215                 }
1216             }
1217           /* External address must be allocated */
1218           if (!a)
1219             return VNET_API_ERROR_NO_SUCH_ENTRY;
1220         }
1221
1222       pool_get (sm->static_mappings, m);
1223       clib_memset (m, 0, sizeof (*m));
1224       m->tag = vec_dup (tag);
1225       m->external_addr = e_addr;
1226       m->external_port = e_port;
1227       m->proto = proto;
1228       m->twice_nat = twice_nat;
1229       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1230       if (out2in_only)
1231         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1232       m->affinity = affinity;
1233
1234       if (affinity)
1235         m->affinity_per_service_list_head_index =
1236           nat_affinity_get_per_service_list_head_index ();
1237       else
1238         m->affinity_per_service_list_head_index = ~0;
1239
1240       m_key.addr = m->external_addr;
1241       m_key.port = m->external_port;
1242       m_key.protocol = m->proto;
1243       m_key.fib_index = 0;
1244       kv.key = m_key.as_u64;
1245       kv.value = m - sm->static_mappings;
1246       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1247         {
1248           nat_log_err ("static_mapping_by_external key add failed");
1249           return VNET_API_ERROR_UNSPECIFIED;
1250         }
1251
1252       m_key.fib_index = m->fib_index;
1253       for (i = 0; i < vec_len (locals); i++)
1254         {
1255           locals[i].fib_index =
1256             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1257                                                locals[i].vrf_id,
1258                                                FIB_SOURCE_PLUGIN_LOW);
1259           m_key.addr = locals[i].addr;
1260           m_key.fib_index = locals[i].fib_index;
1261           if (!out2in_only)
1262             {
1263               m_key.port = locals[i].port;
1264               kv.key = m_key.as_u64;
1265               kv.value = m - sm->static_mappings;
1266               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1267             }
1268           locals[i].prefix = (i == 0) ? locals[i].probability :
1269             (locals[i - 1].prefix + locals[i].probability);
1270           pool_get (m->locals, local);
1271           *local = locals[i];
1272           if (sm->num_workers > 1)
1273             {
1274               ip4_header_t ip = {
1275                 .src_address = locals[i].addr,
1276               };
1277               bitmap =
1278                 clib_bitmap_set (bitmap,
1279                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1280             }
1281         }
1282
1283       /* Assign workers */
1284       if (sm->num_workers > 1)
1285         {
1286           /* *INDENT-OFF* */
1287           clib_bitmap_foreach (i, bitmap,
1288             ({
1289                vec_add1(m->workers, i);
1290             }));
1291           /* *INDENT-ON* */
1292         }
1293     }
1294   else
1295     {
1296       if (!m)
1297         return VNET_API_ERROR_NO_SUCH_ENTRY;
1298
1299       if (!is_lb_static_mapping (m))
1300         return VNET_API_ERROR_INVALID_VALUE;
1301
1302       /* Free external address port */
1303       if (!(sm->static_mapping_only || out2in_only))
1304         {
1305           for (i = 0; i < vec_len (sm->addresses); i++)
1306             {
1307               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1308                 {
1309                   a = sm->addresses + i;
1310                   switch (proto)
1311                     {
1312 #define _(N, j, n, s) \
1313                     case SNAT_PROTOCOL_##N: \
1314                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1315                       if (e_port > 1024) \
1316                         { \
1317                           a->busy_##n##_ports--; \
1318                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1319                         } \
1320                       break;
1321                       foreach_snat_protocol
1322 #undef _
1323                     default:
1324                       nat_log_info ("unknown protocol");
1325                       return VNET_API_ERROR_INVALID_VALUE_2;
1326                     }
1327                   break;
1328                 }
1329             }
1330         }
1331
1332       m_key.addr = m->external_addr;
1333       m_key.port = m->external_port;
1334       m_key.protocol = m->proto;
1335       m_key.fib_index = 0;
1336       kv.key = m_key.as_u64;
1337       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1338         {
1339           nat_log_err ("static_mapping_by_external key del failed");
1340           return VNET_API_ERROR_UNSPECIFIED;
1341         }
1342
1343       /* *INDENT-OFF* */
1344       pool_foreach (local, m->locals,
1345       ({
1346           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1347                             FIB_SOURCE_PLUGIN_LOW);
1348           m_key.addr = local->addr;
1349           if (!out2in_only)
1350             {
1351               m_key.port = local->port;
1352               m_key.fib_index = local->fib_index;
1353               kv.key = m_key.as_u64;
1354               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1355                 {
1356                   nat_log_err ("static_mapping_by_local key del failed");
1357                   return VNET_API_ERROR_UNSPECIFIED;
1358                 }
1359             }
1360
1361           if (sm->num_workers > 1)
1362             {
1363               ip4_header_t ip = {
1364                 .src_address = local->addr,
1365               };
1366               tsm = vec_elt_at_index (sm->per_thread_data,
1367                                       sm->worker_in2out_cb (&ip, m->fib_index));
1368             }
1369           else
1370             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1371
1372           /* Delete sessions */
1373           u_key.addr = local->addr;
1374           u_key.fib_index = local->fib_index;
1375           kv.key = u_key.as_u64;
1376           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1377             {
1378               u = pool_elt_at_index (tsm->users, value.value);
1379               if (u->nstaticsessions)
1380                 {
1381                   head_index = u->sessions_per_user_list_head_index;
1382                   head = pool_elt_at_index (tsm->list_pool, head_index);
1383                   elt_index = head->next;
1384                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1385                   ses_index = elt->value;
1386                   while (ses_index != ~0)
1387                     {
1388                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1389                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1390                       ses_index = elt->value;
1391
1392                       if (!(is_lb_session (s)))
1393                         continue;
1394
1395                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1396                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1397                         continue;
1398
1399                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1400                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1401                     }
1402                 }
1403             }
1404       }));
1405       /* *INDENT-ON* */
1406       if (m->affinity)
1407         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1408       pool_free (m->locals);
1409       vec_free (m->tag);
1410       vec_free (m->workers);
1411
1412       pool_put (sm->static_mappings, m);
1413     }
1414
1415   return 0;
1416 }
1417
1418 int
1419 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1420                                        ip4_address_t l_addr, u16 l_port,
1421                                        snat_protocol_t proto, u32 vrf_id,
1422                                        u8 probability, u8 is_add)
1423 {
1424   snat_main_t *sm = &snat_main;
1425   snat_static_mapping_t *m = 0;
1426   snat_session_key_t m_key;
1427   clib_bihash_kv_8_8_t kv, value;
1428   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1429   snat_main_per_thread_data_t *tsm;
1430   snat_user_key_t u_key;
1431   snat_user_t *u;
1432   snat_session_t *s;
1433   dlist_elt_t *head, *elt;
1434   u32 elt_index, head_index, ses_index, *locals = 0;
1435   uword *bitmap = 0;
1436   int i;
1437
1438   if (!sm->endpoint_dependent)
1439     return VNET_API_ERROR_FEATURE_DISABLED;
1440
1441   m_key.addr = e_addr;
1442   m_key.port = e_port;
1443   m_key.protocol = proto;
1444   m_key.fib_index = 0;
1445   kv.key = m_key.as_u64;
1446   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1447     m = pool_elt_at_index (sm->static_mappings, value.value);
1448
1449   if (!m)
1450     return VNET_API_ERROR_NO_SUCH_ENTRY;
1451
1452   if (!is_lb_static_mapping (m))
1453     return VNET_API_ERROR_INVALID_VALUE;
1454
1455   /* *INDENT-OFF* */
1456   pool_foreach (local, m->locals,
1457   ({
1458     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1459         (local->vrf_id == vrf_id))
1460       {
1461         match_local = local;
1462         break;
1463       }
1464   }));
1465   /* *INDENT-ON* */
1466
1467   if (is_add)
1468     {
1469       if (match_local)
1470         return VNET_API_ERROR_VALUE_EXIST;
1471
1472       pool_get (m->locals, local);
1473       clib_memset (local, 0, sizeof (*local));
1474       local->addr.as_u32 = l_addr.as_u32;
1475       local->port = l_port;
1476       local->probability = probability;
1477       local->vrf_id = vrf_id;
1478       local->fib_index =
1479         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1480                                            FIB_SOURCE_PLUGIN_LOW);
1481
1482       if (!is_out2in_only_static_mapping (m))
1483         {
1484           m_key.addr = l_addr;
1485           m_key.port = l_port;
1486           m_key.fib_index = local->fib_index;
1487           kv.key = m_key.as_u64;
1488           kv.value = m - sm->static_mappings;
1489           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1490             nat_log_err ("static_mapping_by_local key add failed");
1491         }
1492     }
1493   else
1494     {
1495       if (!match_local)
1496         return VNET_API_ERROR_NO_SUCH_ENTRY;
1497
1498       if (pool_elts (m->locals) < 3)
1499         return VNET_API_ERROR_UNSPECIFIED;
1500
1501       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1502                         FIB_SOURCE_PLUGIN_LOW);
1503
1504       if (!is_out2in_only_static_mapping (m))
1505         {
1506           m_key.addr = l_addr;
1507           m_key.port = l_port;
1508           m_key.fib_index = match_local->fib_index;
1509           kv.key = m_key.as_u64;
1510           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1511             nat_log_err ("static_mapping_by_local key del failed");
1512         }
1513
1514       if (sm->num_workers > 1)
1515         {
1516           ip4_header_t ip = {
1517             .src_address = local->addr,
1518           };
1519           tsm = vec_elt_at_index (sm->per_thread_data,
1520                                   sm->worker_in2out_cb (&ip, m->fib_index));
1521         }
1522       else
1523         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1524
1525       /* Delete sessions */
1526       u_key.addr = match_local->addr;
1527       u_key.fib_index = match_local->fib_index;
1528       kv.key = u_key.as_u64;
1529       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1530         {
1531           u = pool_elt_at_index (tsm->users, value.value);
1532           if (u->nstaticsessions)
1533             {
1534               head_index = u->sessions_per_user_list_head_index;
1535               head = pool_elt_at_index (tsm->list_pool, head_index);
1536               elt_index = head->next;
1537               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1538               ses_index = elt->value;
1539               while (ses_index != ~0)
1540                 {
1541                   s = pool_elt_at_index (tsm->sessions, ses_index);
1542                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1543                   ses_index = elt->value;
1544
1545                   if (!(is_lb_session (s)))
1546                     continue;
1547
1548                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1549                       (clib_net_to_host_u16 (s->in2out.port) !=
1550                        match_local->port))
1551                     continue;
1552
1553                   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1554                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1555                 }
1556             }
1557         }
1558
1559       pool_put (m->locals, match_local);
1560     }
1561
1562   vec_free (m->workers);
1563
1564   /* *INDENT-OFF* */
1565   pool_foreach (local, m->locals,
1566   ({
1567     vec_add1 (locals, local - m->locals);
1568     if (sm->num_workers > 1)
1569       {
1570         ip4_header_t ip;
1571         ip.src_address.as_u32 = local->addr.as_u32,
1572         bitmap = clib_bitmap_set (bitmap,
1573                                   sm->worker_in2out_cb (&ip, local->fib_index),
1574                                   1);
1575       }
1576   }));
1577   /* *INDENT-ON* */
1578
1579   ASSERT (vec_len (locals) > 1);
1580
1581   local = pool_elt_at_index (m->locals, locals[0]);
1582   local->prefix = local->probability;
1583   for (i = 1; i < vec_len (locals); i++)
1584     {
1585       local = pool_elt_at_index (m->locals, locals[i]);
1586       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1587       local->prefix = local->probability + prev_local->prefix;
1588     }
1589
1590   /* Assign workers */
1591   if (sm->num_workers > 1)
1592     {
1593       /* *INDENT-OFF* */
1594       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1595       /* *INDENT-ON* */
1596     }
1597
1598   return 0;
1599 }
1600
1601 int
1602 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1603                   u8 twice_nat)
1604 {
1605   snat_address_t *a = 0;
1606   snat_session_t *ses;
1607   u32 *ses_to_be_removed = 0, *ses_index;
1608   snat_main_per_thread_data_t *tsm;
1609   snat_static_mapping_t *m;
1610   snat_interface_t *interface;
1611   int i;
1612   snat_address_t *addresses =
1613     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1614
1615   /* Find SNAT address */
1616   for (i = 0; i < vec_len (addresses); i++)
1617     {
1618       if (addresses[i].addr.as_u32 == addr.as_u32)
1619         {
1620           a = addresses + i;
1621           break;
1622         }
1623     }
1624   if (!a)
1625     return VNET_API_ERROR_NO_SUCH_ENTRY;
1626
1627   if (delete_sm)
1628     {
1629       /* *INDENT-OFF* */
1630       pool_foreach (m, sm->static_mappings,
1631       ({
1632           if (m->external_addr.as_u32 == addr.as_u32)
1633             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1634                                             m->local_port, m->external_port,
1635                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1636                                             m->proto, 0, m->twice_nat,
1637                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1638       }));
1639       /* *INDENT-ON* */
1640     }
1641   else
1642     {
1643       /* Check if address is used in some static mapping */
1644       if (is_snat_address_used_in_static_mapping (sm, addr))
1645         {
1646           nat_log_notice ("address used in static mapping");
1647           return VNET_API_ERROR_UNSPECIFIED;
1648         }
1649     }
1650
1651   if (a->fib_index != ~0)
1652     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1653
1654   /* Delete sessions using address */
1655   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1656     {
1657       /* *INDENT-OFF* */
1658       vec_foreach (tsm, sm->per_thread_data)
1659         {
1660           pool_foreach (ses, tsm->sessions, ({
1661             if (ses->out2in.addr.as_u32 == addr.as_u32)
1662               {
1663                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1664                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1665               }
1666           }));
1667
1668           vec_foreach (ses_index, ses_to_be_removed)
1669             {
1670               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1671               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1672             }
1673
1674           vec_free (ses_to_be_removed);
1675         }
1676       /* *INDENT-ON* */
1677     }
1678
1679 #define _(N, i, n, s) \
1680   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1681   vec_free (a->busy_##n##_ports_per_thread);
1682   foreach_snat_protocol
1683 #undef _
1684     if (twice_nat)
1685     {
1686       vec_del1 (sm->twice_nat_addresses, i);
1687       return 0;
1688     }
1689   else
1690     vec_del1 (sm->addresses, i);
1691
1692   /* Delete external address from FIB */
1693   /* *INDENT-OFF* */
1694   pool_foreach (interface, sm->interfaces,
1695   ({
1696     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1697       continue;
1698
1699     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1700     break;
1701   }));
1702   pool_foreach (interface, sm->output_feature_interfaces,
1703   ({
1704     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1705       continue;
1706
1707     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1708     break;
1709   }));
1710   /* *INDENT-ON* */
1711
1712   return 0;
1713 }
1714
1715 int
1716 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1717 {
1718   snat_main_t *sm = &snat_main;
1719   snat_interface_t *i;
1720   const char *feature_name, *del_feature_name;
1721   snat_address_t *ap;
1722   snat_static_mapping_t *m;
1723   snat_det_map_t *dm;
1724   nat_outside_fib_t *outside_fib;
1725   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1726                                                        sw_if_index);
1727
1728   if (sm->out2in_dpo && !is_inside)
1729     return VNET_API_ERROR_UNSUPPORTED;
1730
1731   /* *INDENT-OFF* */
1732   pool_foreach (i, sm->output_feature_interfaces,
1733   ({
1734     if (i->sw_if_index == sw_if_index)
1735       return VNET_API_ERROR_VALUE_EXIST;
1736   }));
1737   /* *INDENT-ON* */
1738
1739   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1740     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1741   else
1742     {
1743       if (sm->num_workers > 1 && !sm->deterministic)
1744         feature_name =
1745           is_inside ? "nat44-in2out-worker-handoff" :
1746           "nat44-out2in-worker-handoff";
1747       else if (sm->deterministic)
1748         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1749       else if (sm->endpoint_dependent)
1750         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1751       else
1752         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1753     }
1754
1755   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1756     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1757                                                       NAT_FQ_NELTS);
1758
1759   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1760     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1761                                                       NAT_FQ_NELTS);
1762
1763   if (!is_inside)
1764     {
1765       /* *INDENT-OFF* */
1766       vec_foreach (outside_fib, sm->outside_fibs)
1767         {
1768           if (outside_fib->fib_index == fib_index)
1769             {
1770               if (is_del)
1771                 {
1772                   outside_fib->refcount--;
1773                   if (!outside_fib->refcount)
1774                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1775                 }
1776               else
1777                 outside_fib->refcount++;
1778               goto feature_set;
1779             }
1780         }
1781       /* *INDENT-ON* */
1782       if (!is_del)
1783         {
1784           vec_add2 (sm->outside_fibs, outside_fib, 1);
1785           outside_fib->refcount = 1;
1786           outside_fib->fib_index = fib_index;
1787         }
1788     }
1789 feature_set:
1790   /* *INDENT-OFF* */
1791   pool_foreach (i, sm->interfaces,
1792   ({
1793     if (i->sw_if_index == sw_if_index)
1794       {
1795         if (is_del)
1796           {
1797             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1798               {
1799                 if (is_inside)
1800                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1801                 else
1802                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1803
1804                 if (sm->num_workers > 1 && !sm->deterministic)
1805                   {
1806                     del_feature_name = "nat44-handoff-classify";
1807                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1808                                                  "nat44-out2in-worker-handoff";
1809                   }
1810                 else if (sm->deterministic)
1811                   {
1812                     del_feature_name = "nat44-det-classify";
1813                     feature_name = !is_inside ?  "nat44-det-in2out" :
1814                                                  "nat44-det-out2in";
1815                   }
1816                 else if (sm->endpoint_dependent)
1817                   {
1818                     del_feature_name = "nat44-ed-classify";
1819                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1820                                                  "nat44-ed-out2in";
1821                   }
1822                 else
1823                   {
1824                     del_feature_name = "nat44-classify";
1825                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1826                   }
1827
1828                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1829                                              sw_if_index, 0, 0, 0);
1830                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1831                                              sw_if_index, 1, 0, 0);
1832                 if (!is_inside)
1833                   {
1834                     if (sm->endpoint_dependent)
1835                       vnet_feature_enable_disable ("ip4-local",
1836                                                    "nat44-ed-hairpinning",
1837                                                    sw_if_index, 1, 0, 0);
1838                     else if (!sm->deterministic)
1839                       vnet_feature_enable_disable ("ip4-local",
1840                                                    "nat44-hairpinning",
1841                                                    sw_if_index, 1, 0, 0);
1842                   }
1843               }
1844             else
1845               {
1846                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1847                                              sw_if_index, 0, 0, 0);
1848                 pool_put (sm->interfaces, i);
1849                 if (is_inside)
1850                   {
1851                     if (sm->endpoint_dependent)
1852                       vnet_feature_enable_disable ("ip4-local",
1853                                                    "nat44-ed-hairpinning",
1854                                                    sw_if_index, 0, 0, 0);
1855                     else if (!sm->deterministic)
1856                       vnet_feature_enable_disable ("ip4-local",
1857                                                    "nat44-hairpinning",
1858                                                    sw_if_index, 0, 0, 0);
1859                   }
1860               }
1861           }
1862         else
1863           {
1864             if ((nat_interface_is_inside(i) && is_inside) ||
1865                 (nat_interface_is_outside(i) && !is_inside))
1866               return 0;
1867
1868             if (sm->num_workers > 1 && !sm->deterministic)
1869               {
1870                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1871                                                  "nat44-out2in-worker-handoff";
1872                 feature_name = "nat44-handoff-classify";
1873               }
1874             else if (sm->deterministic)
1875               {
1876                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1877                                                  "nat44-det-out2in";
1878                 feature_name = "nat44-det-classify";
1879               }
1880             else if (sm->endpoint_dependent)
1881               {
1882                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1883                                                  "nat44-ed-out2in";
1884                 feature_name = "nat44-ed-classify";
1885               }
1886             else
1887               {
1888                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1889                 feature_name = "nat44-classify";
1890               }
1891
1892             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1893                                          sw_if_index, 0, 0, 0);
1894             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1895                                          sw_if_index, 1, 0, 0);
1896             if (!is_inside)
1897               {
1898                 if (sm->endpoint_dependent)
1899                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1900                                                sw_if_index, 0, 0, 0);
1901                 else if (!sm->deterministic)
1902                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1903                                                sw_if_index, 0, 0, 0);
1904               }
1905             goto set_flags;
1906           }
1907
1908         goto fib;
1909       }
1910   }));
1911   /* *INDENT-ON* */
1912
1913   if (is_del)
1914     return VNET_API_ERROR_NO_SUCH_ENTRY;
1915
1916   pool_get (sm->interfaces, i);
1917   i->sw_if_index = sw_if_index;
1918   i->flags = 0;
1919   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1920                                0);
1921
1922   if (is_inside && !sm->out2in_dpo)
1923     {
1924       if (sm->endpoint_dependent)
1925         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1926                                      sw_if_index, 1, 0, 0);
1927       else if (!sm->deterministic)
1928         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1929                                      sw_if_index, 1, 0, 0);
1930     }
1931
1932 set_flags:
1933   if (is_inside)
1934     {
1935       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1936       return 0;
1937     }
1938   else
1939     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1940
1941   /* Add/delete external addresses to FIB */
1942 fib:
1943   /* *INDENT-OFF* */
1944   vec_foreach (ap, sm->addresses)
1945     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1946
1947   pool_foreach (m, sm->static_mappings,
1948   ({
1949     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1950       continue;
1951
1952     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1953   }));
1954
1955   pool_foreach (dm, sm->det_maps,
1956   ({
1957     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1958   }));
1959   /* *INDENT-ON* */
1960
1961   return 0;
1962 }
1963
1964 int
1965 snat_interface_add_del_output_feature (u32 sw_if_index,
1966                                        u8 is_inside, int is_del)
1967 {
1968   snat_main_t *sm = &snat_main;
1969   snat_interface_t *i;
1970   snat_address_t *ap;
1971   snat_static_mapping_t *m;
1972   nat_outside_fib_t *outside_fib;
1973   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1974                                                        sw_if_index);
1975
1976
1977   if (sm->deterministic ||
1978       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1979     return VNET_API_ERROR_UNSUPPORTED;
1980
1981   /* *INDENT-OFF* */
1982   pool_foreach (i, sm->interfaces,
1983   ({
1984     if (i->sw_if_index == sw_if_index)
1985       return VNET_API_ERROR_VALUE_EXIST;
1986   }));
1987   /* *INDENT-ON* */
1988
1989   if (!is_inside)
1990     {
1991       /* *INDENT-OFF* */
1992       vec_foreach (outside_fib, sm->outside_fibs)
1993         {
1994           if (outside_fib->fib_index == fib_index)
1995             {
1996               if (is_del)
1997                 {
1998                   outside_fib->refcount--;
1999                   if (!outside_fib->refcount)
2000                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2001                 }
2002               else
2003                 outside_fib->refcount++;
2004               goto feature_set;
2005             }
2006         }
2007       /* *INDENT-ON* */
2008       if (!is_del)
2009         {
2010           vec_add2 (sm->outside_fibs, outside_fib, 1);
2011           outside_fib->refcount = 1;
2012           outside_fib->fib_index = fib_index;
2013         }
2014     }
2015
2016 feature_set:
2017   if (is_inside)
2018     {
2019       if (sm->endpoint_dependent)
2020         {
2021           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2022                                        sw_if_index, !is_del, 0, 0);
2023           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2024                                        sw_if_index, !is_del, 0, 0);
2025         }
2026       else
2027         {
2028           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2029                                        sw_if_index, !is_del, 0, 0);
2030           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2031                                        sw_if_index, !is_del, 0, 0);
2032         }
2033       goto fq;
2034     }
2035
2036   if (sm->num_workers > 1)
2037     {
2038       vnet_feature_enable_disable ("ip4-unicast",
2039                                    "nat44-out2in-worker-handoff",
2040                                    sw_if_index, !is_del, 0, 0);
2041       vnet_feature_enable_disable ("ip4-output",
2042                                    "nat44-in2out-output-worker-handoff",
2043                                    sw_if_index, !is_del, 0, 0);
2044     }
2045   else
2046     {
2047       if (sm->endpoint_dependent)
2048         {
2049           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2050                                        sw_if_index, !is_del, 0, 0);
2051           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2052                                        sw_if_index, !is_del, 0, 0);
2053         }
2054       else
2055         {
2056           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2057                                        sw_if_index, !is_del, 0, 0);
2058           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2059                                        sw_if_index, !is_del, 0, 0);
2060         }
2061     }
2062
2063 fq:
2064   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2065     sm->fq_in2out_output_index =
2066       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2067
2068   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2069     sm->fq_out2in_index =
2070       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2071
2072   /* *INDENT-OFF* */
2073   pool_foreach (i, sm->output_feature_interfaces,
2074   ({
2075     if (i->sw_if_index == sw_if_index)
2076       {
2077         if (is_del)
2078           pool_put (sm->output_feature_interfaces, i);
2079         else
2080           return VNET_API_ERROR_VALUE_EXIST;
2081
2082         goto fib;
2083       }
2084   }));
2085   /* *INDENT-ON* */
2086
2087   if (is_del)
2088     return VNET_API_ERROR_NO_SUCH_ENTRY;
2089
2090   pool_get (sm->output_feature_interfaces, i);
2091   i->sw_if_index = sw_if_index;
2092   i->flags = 0;
2093   if (is_inside)
2094     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2095   else
2096     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2097
2098   /* Add/delete external addresses to FIB */
2099 fib:
2100   if (is_inside)
2101     return 0;
2102
2103   /* *INDENT-OFF* */
2104   vec_foreach (ap, sm->addresses)
2105     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2106
2107   pool_foreach (m, sm->static_mappings,
2108   ({
2109     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2110       continue;
2111
2112     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2113   }));
2114   /* *INDENT-ON* */
2115
2116   return 0;
2117 }
2118
2119 int
2120 snat_set_workers (uword * bitmap)
2121 {
2122   snat_main_t *sm = &snat_main;
2123   int i, j = 0;
2124
2125   if (sm->num_workers < 2)
2126     return VNET_API_ERROR_FEATURE_DISABLED;
2127
2128   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2129     return VNET_API_ERROR_INVALID_WORKER;
2130
2131   vec_free (sm->workers);
2132   /* *INDENT-OFF* */
2133   clib_bitmap_foreach (i, bitmap,
2134     ({
2135       vec_add1(sm->workers, i);
2136       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2137       j++;
2138     }));
2139   /* *INDENT-ON* */
2140
2141   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2142   sm->num_snat_thread = _vec_len (sm->workers);
2143
2144   return 0;
2145 }
2146
2147 static void
2148 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2149                          u32 old_fib_index)
2150 {
2151   snat_main_t *sm = &snat_main;
2152   nat_outside_fib_t *outside_fib;
2153   snat_interface_t *i;
2154   u8 is_add = 1;
2155
2156   if (new_fib_index == old_fib_index)
2157     return;
2158
2159   if (!vec_len (sm->outside_fibs))
2160     return;
2161
2162   pool_foreach (i, sm->interfaces, (
2163                                      {
2164                                      if (i->sw_if_index == sw_if_index)
2165                                      {
2166                                      if (!(nat_interface_is_outside (i)))
2167                                      return;}
2168                                      }
2169                 ));
2170   vec_foreach (outside_fib, sm->outside_fibs)
2171   {
2172     if (outside_fib->fib_index == old_fib_index)
2173       {
2174         outside_fib->refcount--;
2175         if (!outside_fib->refcount)
2176           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2177         break;
2178       }
2179   }
2180
2181   vec_foreach (outside_fib, sm->outside_fibs)
2182   {
2183     if (outside_fib->fib_index == new_fib_index)
2184       {
2185         outside_fib->refcount++;
2186         is_add = 0;
2187         break;
2188       }
2189   }
2190
2191   if (is_add)
2192     {
2193       vec_add2 (sm->outside_fibs, outside_fib, 1);
2194       outside_fib->refcount = 1;
2195       outside_fib->fib_index = new_fib_index;
2196     }
2197 }
2198
2199 static void
2200 snat_ip4_table_bind (ip4_main_t * im,
2201                      uword opaque,
2202                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2203 {
2204   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2205 }
2206
2207 static void
2208 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2209                                        uword opaque,
2210                                        u32 sw_if_index,
2211                                        ip4_address_t * address,
2212                                        u32 address_length,
2213                                        u32 if_address_index, u32 is_delete);
2214
2215 static void
2216 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2217                                  uword opaque,
2218                                  u32 sw_if_index,
2219                                  ip4_address_t * address,
2220                                  u32 address_length,
2221                                  u32 if_address_index, u32 is_delete);
2222
2223 static int
2224 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2225                                  u32 fib_index,
2226                                  u32 thread_index,
2227                                  snat_session_key_t * k,
2228                                  u16 port_per_thread, u32 snat_thread_index);
2229
2230 static clib_error_t *
2231 snat_init (vlib_main_t * vm)
2232 {
2233   snat_main_t *sm = &snat_main;
2234   clib_error_t *error = 0;
2235   ip4_main_t *im = &ip4_main;
2236   ip_lookup_main_t *lm = &im->lookup_main;
2237   uword *p;
2238   vlib_thread_registration_t *tr;
2239   vlib_thread_main_t *tm = vlib_get_thread_main ();
2240   uword *bitmap = 0;
2241   u32 i;
2242   ip4_add_del_interface_address_callback_t cb4;
2243   vlib_node_t *error_drop_node;
2244
2245   sm->vlib_main = vm;
2246   sm->vnet_main = vnet_get_main ();
2247   sm->ip4_main = im;
2248   sm->ip4_lookup_main = lm;
2249   sm->api_main = &api_main;
2250   sm->first_worker_index = 0;
2251   sm->num_workers = 0;
2252   sm->num_snat_thread = 1;
2253   sm->workers = 0;
2254   sm->port_per_thread = 0xffff - 1024;
2255   sm->fq_in2out_index = ~0;
2256   sm->fq_out2in_index = ~0;
2257   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2258   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2259   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2260   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2261   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2262   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2263   sm->forwarding_enabled = 0;
2264   sm->log_class = vlib_log_register_class ("nat", 0);
2265   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2266   sm->error_node_index = error_drop_node->index;
2267   sm->mss_clamping = 0;
2268
2269   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2270   if (p)
2271     {
2272       tr = (vlib_thread_registration_t *) p[0];
2273       if (tr)
2274         {
2275           sm->num_workers = tr->count;
2276           sm->first_worker_index = tr->first_index;
2277         }
2278     }
2279
2280   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2281
2282   /* Use all available workers by default */
2283   if (sm->num_workers > 1)
2284     {
2285       for (i = 0; i < sm->num_workers; i++)
2286         bitmap = clib_bitmap_set (bitmap, i, 1);
2287       snat_set_workers (bitmap);
2288       clib_bitmap_free (bitmap);
2289     }
2290   else
2291     {
2292       sm->per_thread_data[0].snat_thread_index = 0;
2293     }
2294
2295   error = snat_api_init (vm, sm);
2296   if (error)
2297     return error;
2298
2299   /* Set up the interface address add/del callback */
2300   cb4.function = snat_ip4_add_del_interface_address_cb;
2301   cb4.function_opaque = 0;
2302
2303   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2304
2305   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2306   cb4.function_opaque = 0;
2307
2308   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2309
2310   nat_dpo_module_init ();
2311
2312   /* Init counters */
2313   sm->total_users.name = "total-users";
2314   sm->total_users.stat_segment_name = "/nat44/total-users";
2315   vlib_validate_simple_counter (&sm->total_users, 0);
2316   vlib_zero_simple_counter (&sm->total_users, 0);
2317   sm->total_sessions.name = "total-sessions";
2318   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2319   vlib_validate_simple_counter (&sm->total_sessions, 0);
2320   vlib_zero_simple_counter (&sm->total_sessions, 0);
2321
2322   /* Init IPFIX logging */
2323   snat_ipfix_logging_init (vm);
2324
2325   /* Init NAT64 */
2326   error = nat64_init (vm);
2327   if (error)
2328     return error;
2329
2330   dslite_init (vm);
2331
2332   nat66_init ();
2333
2334   ip4_table_bind_callback_t cbt4 = {
2335     .function = snat_ip4_table_bind,
2336   };
2337   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2338
2339   /* Init virtual fragmenentation reassembly */
2340   return nat_reass_init (vm);
2341 }
2342
2343 VLIB_INIT_FUNCTION (snat_init);
2344
2345 void
2346 snat_free_outside_address_and_port (snat_address_t * addresses,
2347                                     u32 thread_index, snat_session_key_t * k)
2348 {
2349   snat_address_t *a;
2350   u32 address_index;
2351   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2352
2353   for (address_index = 0; address_index < vec_len (addresses);
2354        address_index++)
2355     {
2356       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2357         break;
2358     }
2359
2360   ASSERT (address_index < vec_len (addresses));
2361
2362   a = addresses + address_index;
2363
2364   switch (k->protocol)
2365     {
2366 #define _(N, i, n, s) \
2367     case SNAT_PROTOCOL_##N: \
2368       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2369         port_host_byte_order) == 1); \
2370       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2371         port_host_byte_order, 0); \
2372       a->busy_##n##_ports--; \
2373       a->busy_##n##_ports_per_thread[thread_index]--; \
2374       break;
2375       foreach_snat_protocol
2376 #undef _
2377     default:
2378       nat_log_info ("unknown protocol");
2379       return;
2380     }
2381 }
2382
2383 int
2384 snat_static_mapping_match (snat_main_t * sm,
2385                            snat_session_key_t match,
2386                            snat_session_key_t * mapping,
2387                            u8 by_external,
2388                            u8 * is_addr_only,
2389                            twice_nat_type_t * twice_nat,
2390                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2391                            u8 * is_identity_nat)
2392 {
2393   clib_bihash_kv_8_8_t kv, value;
2394   snat_static_mapping_t *m;
2395   snat_session_key_t m_key;
2396   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2397   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2398   u8 backend_index;
2399   nat44_lb_addr_port_t *local;
2400
2401   m_key.fib_index = match.fib_index;
2402   if (by_external)
2403     {
2404       mapping_hash = &sm->static_mapping_by_external;
2405       m_key.fib_index = 0;
2406     }
2407
2408   m_key.addr = match.addr;
2409   m_key.port = clib_net_to_host_u16 (match.port);
2410   m_key.protocol = match.protocol;
2411
2412   kv.key = m_key.as_u64;
2413
2414   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2415     {
2416       /* Try address only mapping */
2417       m_key.port = 0;
2418       m_key.protocol = 0;
2419       kv.key = m_key.as_u64;
2420       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2421         return 1;
2422     }
2423
2424   m = pool_elt_at_index (sm->static_mappings, value.value);
2425
2426   if (by_external)
2427     {
2428       if (is_lb_static_mapping (m))
2429         {
2430           if (PREDICT_FALSE (lb != 0))
2431             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2432           if (m->affinity)
2433             {
2434               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2435                                               match.protocol, match.port,
2436                                               &backend_index))
2437                 goto get_local;
2438
2439               local = pool_elt_at_index (m->locals, backend_index);
2440               mapping->addr = local->addr;
2441               mapping->port = clib_host_to_net_u16 (local->port);
2442               mapping->fib_index = local->fib_index;
2443               goto end;
2444             }
2445         get_local:
2446           /* *INDENT-OFF* */
2447           pool_foreach_index (i, m->locals,
2448           ({
2449             vec_add1 (tmp, i);
2450           }));
2451           /* *INDENT-ON* */
2452           hi = vec_len (tmp) - 1;
2453           local = pool_elt_at_index (m->locals, tmp[hi]);
2454           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2455           while (lo < hi)
2456             {
2457               mid = ((hi - lo) >> 1) + lo;
2458               local = pool_elt_at_index (m->locals, tmp[mid]);
2459               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2460             }
2461           local = pool_elt_at_index (m->locals, tmp[lo]);
2462           if (!(local->prefix >= rand))
2463             return 1;
2464           if (PREDICT_FALSE (sm->num_workers > 1))
2465             {
2466               ip4_header_t ip = {
2467                 .src_address = local->addr,
2468               };
2469               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2470                   vlib_get_thread_index ())
2471                 goto get_local;
2472             }
2473           mapping->addr = local->addr;
2474           mapping->port = clib_host_to_net_u16 (local->port);
2475           mapping->fib_index = local->fib_index;
2476           if (m->affinity)
2477             {
2478               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2479                                                 match.protocol, match.port,
2480                                                 tmp[lo], m->affinity,
2481                                                 m->affinity_per_service_list_head_index))
2482                 nat_log_info ("create affinity record failed");
2483             }
2484           vec_free (tmp);
2485         }
2486       else
2487         {
2488           if (PREDICT_FALSE (lb != 0))
2489             *lb = NO_LB_NAT;
2490           mapping->fib_index = m->fib_index;
2491           mapping->addr = m->local_addr;
2492           /* Address only mapping doesn't change port */
2493           mapping->port = is_addr_only_static_mapping (m) ? match.port
2494             : clib_host_to_net_u16 (m->local_port);
2495         }
2496       mapping->protocol = m->proto;
2497     }
2498   else
2499     {
2500       mapping->addr = m->external_addr;
2501       /* Address only mapping doesn't change port */
2502       mapping->port = is_addr_only_static_mapping (m) ? match.port
2503         : clib_host_to_net_u16 (m->external_port);
2504       mapping->fib_index = sm->outside_fib_index;
2505     }
2506
2507 end:
2508   if (PREDICT_FALSE (is_addr_only != 0))
2509     *is_addr_only = is_addr_only_static_mapping (m);
2510
2511   if (PREDICT_FALSE (twice_nat != 0))
2512     *twice_nat = m->twice_nat;
2513
2514   if (PREDICT_FALSE (is_identity_nat != 0))
2515     *is_identity_nat = is_identity_static_mapping (m);
2516
2517   return 0;
2518 }
2519
2520 static_always_inline u16
2521 snat_random_port (u16 min, u16 max)
2522 {
2523   snat_main_t *sm = &snat_main;
2524   return min + random_u32 (&sm->random_seed) /
2525     (random_u32_max () / (max - min + 1) + 1);
2526 }
2527
2528 int
2529 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2530                                      u32 fib_index,
2531                                      u32 thread_index,
2532                                      snat_session_key_t * k,
2533                                      u16 port_per_thread,
2534                                      u32 snat_thread_index)
2535 {
2536   snat_main_t *sm = &snat_main;
2537
2538   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2539                                   port_per_thread, snat_thread_index);
2540 }
2541
2542 static int
2543 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2544                                  u32 fib_index,
2545                                  u32 thread_index,
2546                                  snat_session_key_t * k,
2547                                  u16 port_per_thread, u32 snat_thread_index)
2548 {
2549   int i;
2550   snat_address_t *a, *ga = 0;
2551   u32 portnum;
2552
2553   for (i = 0; i < vec_len (addresses); i++)
2554     {
2555       a = addresses + i;
2556       switch (k->protocol)
2557         {
2558 #define _(N, j, n, s) \
2559         case SNAT_PROTOCOL_##N: \
2560           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2561             { \
2562               if (a->fib_index == fib_index) \
2563                 { \
2564                   while (1) \
2565                     { \
2566                       portnum = (port_per_thread * \
2567                         snat_thread_index) + \
2568                         snat_random_port(1, port_per_thread) + 1024; \
2569                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2570                         continue; \
2571                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2572                       a->busy_##n##_ports_per_thread[thread_index]++; \
2573                       a->busy_##n##_ports++; \
2574                       k->addr = a->addr; \
2575                       k->port = clib_host_to_net_u16(portnum); \
2576                       return 0; \
2577                     } \
2578                 } \
2579               else if (a->fib_index == ~0) \
2580                 { \
2581                   ga = a; \
2582                 } \
2583             } \
2584           break;
2585           foreach_snat_protocol
2586 #undef _
2587         default:
2588           nat_log_info ("unknown protocol");
2589           return 1;
2590         }
2591
2592     }
2593
2594   if (ga)
2595     {
2596       a = ga;
2597       switch (k->protocol)
2598         {
2599 #define _(N, j, n, s) \
2600         case SNAT_PROTOCOL_##N: \
2601           while (1) \
2602             { \
2603               portnum = (port_per_thread * \
2604                 snat_thread_index) + \
2605                 snat_random_port(1, port_per_thread) + 1024; \
2606               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2607                 continue; \
2608               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2609               a->busy_##n##_ports_per_thread[thread_index]++; \
2610               a->busy_##n##_ports++; \
2611               k->addr = a->addr; \
2612               k->port = clib_host_to_net_u16(portnum); \
2613               return 0; \
2614             }
2615           break;
2616           foreach_snat_protocol
2617 #undef _
2618         default:
2619           nat_log_info ("unknown protocol");
2620           return 1;
2621         }
2622     }
2623
2624   /* Totally out of translations to use... */
2625   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2626   return 1;
2627 }
2628
2629 static int
2630 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2631                               u32 fib_index,
2632                               u32 thread_index,
2633                               snat_session_key_t * k,
2634                               u16 port_per_thread, u32 snat_thread_index)
2635 {
2636   snat_main_t *sm = &snat_main;
2637   snat_address_t *a = addresses;
2638   u16 m, ports, portnum, A, j;
2639   m = 16 - (sm->psid_offset + sm->psid_length);
2640   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2641
2642   if (!vec_len (addresses))
2643     goto exhausted;
2644
2645   switch (k->protocol)
2646     {
2647 #define _(N, i, n, s) \
2648     case SNAT_PROTOCOL_##N: \
2649       if (a->busy_##n##_ports < ports) \
2650         { \
2651           while (1) \
2652             { \
2653               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2654               j = snat_random_port(0, pow2_mask(m)); \
2655               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2656               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2657                 continue; \
2658               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2659               a->busy_##n##_ports++; \
2660               k->addr = a->addr; \
2661               k->port = clib_host_to_net_u16 (portnum); \
2662               return 0; \
2663             } \
2664         } \
2665       break;
2666       foreach_snat_protocol
2667 #undef _
2668     default:
2669       nat_log_info ("unknown protocol");
2670       return 1;
2671     }
2672
2673 exhausted:
2674   /* Totally out of translations to use... */
2675   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2676   return 1;
2677 }
2678
2679 static int
2680 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2681                                u32 fib_index,
2682                                u32 thread_index,
2683                                snat_session_key_t * k,
2684                                u16 port_per_thread, u32 snat_thread_index)
2685 {
2686   snat_main_t *sm = &snat_main;
2687   snat_address_t *a = addresses;
2688   u16 portnum, ports;
2689
2690   ports = sm->end_port - sm->start_port + 1;
2691
2692   if (!vec_len (addresses))
2693     goto exhausted;
2694
2695   switch (k->protocol)
2696     {
2697 #define _(N, i, n, s) \
2698     case SNAT_PROTOCOL_##N: \
2699       if (a->busy_##n##_ports < ports) \
2700         { \
2701           while (1) \
2702             { \
2703               portnum = snat_random_port(sm->start_port, sm->end_port); \
2704               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2705                 continue; \
2706               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2707               a->busy_##n##_ports++; \
2708               k->addr = a->addr; \
2709               k->port = clib_host_to_net_u16 (portnum); \
2710               return 0; \
2711             } \
2712         } \
2713       break;
2714       foreach_snat_protocol
2715 #undef _
2716     default:
2717       nat_log_info ("unknown protocol");
2718       return 1;
2719     }
2720
2721 exhausted:
2722   /* Totally out of translations to use... */
2723   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2724   return 1;
2725 }
2726
2727 void
2728 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2729 {
2730   dpo_id_t dpo_v4 = DPO_INVALID;
2731   fib_prefix_t pfx = {
2732     .fp_proto = FIB_PROTOCOL_IP4,
2733     .fp_len = 32,
2734     .fp_addr.ip4.as_u32 = addr.as_u32,
2735   };
2736
2737   if (is_add)
2738     {
2739       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2740       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2741                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2742       dpo_reset (&dpo_v4);
2743     }
2744   else
2745     {
2746       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2747     }
2748 }
2749
2750 u8 *
2751 format_session_kvp (u8 * s, va_list * args)
2752 {
2753   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2754   snat_session_key_t k;
2755
2756   k.as_u64 = v->key;
2757
2758   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2759
2760   return s;
2761 }
2762
2763 u8 *
2764 format_static_mapping_kvp (u8 * s, va_list * args)
2765 {
2766   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2767   snat_session_key_t k;
2768
2769   k.as_u64 = v->key;
2770
2771   s = format (s, "%U static-mapping-index %llu",
2772               format_static_mapping_key, &k, v->value);
2773
2774   return s;
2775 }
2776
2777 u8 *
2778 format_user_kvp (u8 * s, va_list * args)
2779 {
2780   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2781   snat_user_key_t k;
2782
2783   k.as_u64 = v->key;
2784
2785   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2786               k.fib_index, v->value);
2787
2788   return s;
2789 }
2790
2791 u8 *
2792 format_ed_session_kvp (u8 * s, va_list * args)
2793 {
2794   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2795   nat_ed_ses_key_t k;
2796
2797   k.as_u64[0] = v->key[0];
2798   k.as_u64[1] = v->key[1];
2799
2800   s =
2801     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2802             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2803             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2804             format_ip_protocol, k.proto, k.fib_index, v->value);
2805
2806   return s;
2807 }
2808
2809 static u32
2810 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2811 {
2812   snat_main_t *sm = &snat_main;
2813   u32 next_worker_index = 0;
2814   u32 hash;
2815
2816   next_worker_index = sm->first_worker_index;
2817   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2818     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2819
2820   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2821     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2822   else
2823     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2824
2825   return next_worker_index;
2826 }
2827
2828 static u32
2829 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2830 {
2831   snat_main_t *sm = &snat_main;
2832   udp_header_t *udp;
2833   u16 port;
2834   snat_session_key_t m_key;
2835   clib_bihash_kv_8_8_t kv, value;
2836   snat_static_mapping_t *m;
2837   u32 proto;
2838   u32 next_worker_index = 0;
2839
2840   /* first try static mappings without port */
2841   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2842     {
2843       m_key.addr = ip0->dst_address;
2844       m_key.port = 0;
2845       m_key.protocol = 0;
2846       m_key.fib_index = rx_fib_index0;
2847       kv.key = m_key.as_u64;
2848       if (!clib_bihash_search_8_8
2849           (&sm->static_mapping_by_external, &kv, &value))
2850         {
2851           m = pool_elt_at_index (sm->static_mappings, value.value);
2852           return m->workers[0];
2853         }
2854     }
2855
2856   proto = ip_proto_to_snat_proto (ip0->protocol);
2857   udp = ip4_next_header (ip0);
2858   port = udp->dst_port;
2859
2860   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2861     {
2862       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2863         return vlib_get_thread_index ();
2864
2865       nat_reass_ip4_t *reass;
2866       reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2867                                   ip0->fragment_id, ip0->protocol);
2868
2869       if (reass && (reass->thread_index != (u32) ~ 0))
2870         return reass->thread_index;
2871
2872       if (ip4_is_first_fragment (ip0))
2873         {
2874           reass =
2875             nat_ip4_reass_create (ip0->src_address, ip0->dst_address,
2876                                   ip0->fragment_id, ip0->protocol);
2877           if (!reass)
2878             goto no_reass;
2879
2880           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2881             {
2882               m_key.addr = ip0->dst_address;
2883               m_key.port = clib_net_to_host_u16 (port);
2884               m_key.protocol = proto;
2885               m_key.fib_index = rx_fib_index0;
2886               kv.key = m_key.as_u64;
2887               if (!clib_bihash_search_8_8
2888                   (&sm->static_mapping_by_external, &kv, &value))
2889                 {
2890                   m = pool_elt_at_index (sm->static_mappings, value.value);
2891                   reass->thread_index = m->workers[0];
2892                   return reass->thread_index;
2893                 }
2894             }
2895           reass->thread_index = sm->first_worker_index;
2896           reass->thread_index +=
2897             sm->workers[(clib_net_to_host_u16 (port) - 1024) /
2898                         sm->port_per_thread];
2899           return reass->thread_index;
2900         }
2901       else
2902         return vlib_get_thread_index ();
2903     }
2904
2905 no_reass:
2906   /* unknown protocol */
2907   if (PREDICT_FALSE (proto == ~0))
2908     {
2909       /* use current thread */
2910       return vlib_get_thread_index ();
2911     }
2912
2913   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2914     {
2915       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2916       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2917       if (!icmp_is_error_message (icmp))
2918         port = echo->identifier;
2919       else
2920         {
2921           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2922           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2923           void *l4_header = ip4_next_header (inner_ip);
2924           switch (proto)
2925             {
2926             case SNAT_PROTOCOL_ICMP:
2927               icmp = (icmp46_header_t *) l4_header;
2928               echo = (icmp_echo_header_t *) (icmp + 1);
2929               port = echo->identifier;
2930               break;
2931             case SNAT_PROTOCOL_UDP:
2932             case SNAT_PROTOCOL_TCP:
2933               port = ((tcp_udp_header_t *) l4_header)->src_port;
2934               break;
2935             default:
2936               return vlib_get_thread_index ();
2937             }
2938         }
2939     }
2940
2941   /* try static mappings with port */
2942   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2943     {
2944       m_key.addr = ip0->dst_address;
2945       m_key.port = clib_net_to_host_u16 (port);
2946       m_key.protocol = proto;
2947       m_key.fib_index = rx_fib_index0;
2948       kv.key = m_key.as_u64;
2949       if (!clib_bihash_search_8_8
2950           (&sm->static_mapping_by_external, &kv, &value))
2951         {
2952           m = pool_elt_at_index (sm->static_mappings, value.value);
2953           return m->workers[0];
2954         }
2955     }
2956
2957   /* worker by outside port */
2958   next_worker_index = sm->first_worker_index;
2959   next_worker_index +=
2960     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2961   return next_worker_index;
2962 }
2963
2964 static u32
2965 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2966 {
2967   snat_main_t *sm = &snat_main;
2968   clib_bihash_kv_8_8_t kv, value;
2969   u32 proto, next_worker_index = 0;
2970   udp_header_t *udp;
2971   u16 port;
2972   snat_static_mapping_t *m;
2973   u32 hash;
2974
2975   /* first try static mappings without port */
2976   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2977     {
2978       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2979       if (!clib_bihash_search_8_8
2980           (&sm->static_mapping_by_external, &kv, &value))
2981         {
2982           m = pool_elt_at_index (sm->static_mappings, value.value);
2983           return m->workers[0];
2984         }
2985     }
2986
2987   proto = ip_proto_to_snat_proto (ip->protocol);
2988
2989   /* unknown protocol */
2990   if (PREDICT_FALSE (proto == ~0))
2991     {
2992       /* use current thread */
2993       return vlib_get_thread_index ();
2994     }
2995
2996   udp = ip4_next_header (ip);
2997   port = udp->dst_port;
2998
2999   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3000     {
3001       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3002       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3003       if (!icmp_is_error_message (icmp))
3004         port = echo->identifier;
3005       else
3006         {
3007           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3008           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3009           void *l4_header = ip4_next_header (inner_ip);
3010           switch (proto)
3011             {
3012             case SNAT_PROTOCOL_ICMP:
3013               icmp = (icmp46_header_t *) l4_header;
3014               echo = (icmp_echo_header_t *) (icmp + 1);
3015               port = echo->identifier;
3016               break;
3017             case SNAT_PROTOCOL_UDP:
3018             case SNAT_PROTOCOL_TCP:
3019               port = ((tcp_udp_header_t *) l4_header)->src_port;
3020               break;
3021             default:
3022               return vlib_get_thread_index ();
3023             }
3024         }
3025     }
3026
3027   /* try static mappings with port */
3028   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3029     {
3030       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
3031                   clib_net_to_host_u16 (port));
3032       if (!clib_bihash_search_8_8
3033           (&sm->static_mapping_by_external, &kv, &value))
3034         {
3035           m = pool_elt_at_index (sm->static_mappings, value.value);
3036           if (!is_lb_static_mapping (m))
3037             return m->workers[0];
3038
3039           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3040             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3041
3042           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3043             return m->workers[hash & (_vec_len (m->workers) - 1)];
3044           else
3045             return m->workers[hash % _vec_len (m->workers)];
3046         }
3047     }
3048
3049   /* worker by outside port */
3050   next_worker_index = sm->first_worker_index;
3051   next_worker_index +=
3052     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3053
3054   return next_worker_index;
3055 }
3056
3057 static clib_error_t *
3058 snat_config (vlib_main_t * vm, unformat_input_t * input)
3059 {
3060   snat_main_t *sm = &snat_main;
3061   nat66_main_t *nm = &nat66_main;
3062   u32 translation_buckets = 1024;
3063   u32 translation_memory_size = 128 << 20;
3064   u32 user_buckets = 128;
3065   u32 user_memory_size = 64 << 20;
3066   u32 max_translations_per_user = 100;
3067   u32 outside_vrf_id = 0;
3068   u32 outside_ip6_vrf_id = 0;
3069   u32 inside_vrf_id = 0;
3070   u32 static_mapping_buckets = 1024;
3071   u32 static_mapping_memory_size = 64 << 20;
3072   u32 nat64_bib_buckets = 1024;
3073   u32 nat64_bib_memory_size = 128 << 20;
3074   u32 nat64_st_buckets = 2048;
3075   u32 nat64_st_memory_size = 256 << 20;
3076   u8 static_mapping_only = 0;
3077   u8 static_mapping_connection_tracking = 0;
3078   snat_main_per_thread_data_t *tsm;
3079   dslite_main_t *dm = &dslite_main;
3080
3081   sm->deterministic = 0;
3082   sm->out2in_dpo = 0;
3083   sm->endpoint_dependent = 0;
3084
3085   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3086     {
3087       if (unformat
3088           (input, "translation hash buckets %d", &translation_buckets))
3089         ;
3090       else if (unformat (input, "translation hash memory %d",
3091                          &translation_memory_size));
3092       else if (unformat (input, "user hash buckets %d", &user_buckets))
3093         ;
3094       else if (unformat (input, "user hash memory %d", &user_memory_size))
3095         ;
3096       else if (unformat (input, "max translations per user %d",
3097                          &max_translations_per_user))
3098         ;
3099       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3100         ;
3101       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3102         ;
3103       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3104         ;
3105       else if (unformat (input, "static mapping only"))
3106         {
3107           static_mapping_only = 1;
3108           if (unformat (input, "connection tracking"))
3109             static_mapping_connection_tracking = 1;
3110         }
3111       else if (unformat (input, "deterministic"))
3112         sm->deterministic = 1;
3113       else if (unformat (input, "nat64 bib hash buckets %d",
3114                          &nat64_bib_buckets))
3115         ;
3116       else if (unformat (input, "nat64 bib hash memory %d",
3117                          &nat64_bib_memory_size))
3118         ;
3119       else
3120         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3121         ;
3122       else if (unformat (input, "nat64 st hash memory %d",
3123                          &nat64_st_memory_size))
3124         ;
3125       else if (unformat (input, "out2in dpo"))
3126         sm->out2in_dpo = 1;
3127       else if (unformat (input, "dslite ce"))
3128         dslite_set_ce (dm, 1);
3129       else if (unformat (input, "endpoint-dependent"))
3130         sm->endpoint_dependent = 1;
3131       else
3132         return clib_error_return (0, "unknown input '%U'",
3133                                   format_unformat_error, input);
3134     }
3135
3136   if (sm->deterministic && sm->endpoint_dependent)
3137     return clib_error_return (0,
3138                               "deterministic and endpoint-dependent modes are mutually exclusive");
3139
3140   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3141     return clib_error_return (0,
3142                               "static mapping only mode available only for simple nat");
3143
3144   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3145     return clib_error_return (0,
3146                               "out2in dpo mode available only for simple nat");
3147
3148   /* for show commands, etc. */
3149   sm->translation_buckets = translation_buckets;
3150   sm->translation_memory_size = translation_memory_size;
3151   /* do not exceed load factor 10 */
3152   sm->max_translations = 10 * translation_buckets;
3153   sm->user_buckets = user_buckets;
3154   sm->user_memory_size = user_memory_size;
3155   sm->max_translations_per_user = max_translations_per_user;
3156   sm->outside_vrf_id = outside_vrf_id;
3157   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3158                                                              outside_vrf_id,
3159                                                              FIB_SOURCE_PLUGIN_HI);
3160   nm->outside_vrf_id = outside_ip6_vrf_id;
3161   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3162                                                              outside_ip6_vrf_id,
3163                                                              FIB_SOURCE_PLUGIN_HI);
3164   sm->inside_vrf_id = inside_vrf_id;
3165   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3166                                                             inside_vrf_id,
3167                                                             FIB_SOURCE_PLUGIN_HI);
3168   sm->static_mapping_only = static_mapping_only;
3169   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3170
3171   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3172                   nat64_st_memory_size);
3173
3174   if (sm->deterministic)
3175     {
3176       sm->in2out_node_index = snat_det_in2out_node.index;
3177       sm->in2out_output_node_index = ~0;
3178       sm->out2in_node_index = snat_det_out2in_node.index;
3179       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3180       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3181     }
3182   else
3183     {
3184       if (sm->endpoint_dependent)
3185         {
3186           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3187           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3188           sm->in2out_node_index = nat44_ed_in2out_node.index;
3189           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3190           sm->out2in_node_index = nat44_ed_out2in_node.index;
3191           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3192           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3193           nat_affinity_init (vm);
3194         }
3195       else
3196         {
3197           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3198           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3199           sm->in2out_node_index = snat_in2out_node.index;
3200           sm->in2out_output_node_index = snat_in2out_output_node.index;
3201           sm->out2in_node_index = snat_out2in_node.index;
3202           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3203           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3204         }
3205       if (!static_mapping_only ||
3206           (static_mapping_only && static_mapping_connection_tracking))
3207         {
3208           /* *INDENT-OFF* */
3209           vec_foreach (tsm, sm->per_thread_data)
3210             {
3211               if (sm->endpoint_dependent)
3212                 {
3213                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3214                                          translation_buckets,
3215                                          translation_memory_size);
3216                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3217                                                       format_ed_session_kvp);
3218
3219                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3220                                          translation_buckets,
3221                                          translation_memory_size);
3222                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3223                                                       format_ed_session_kvp);
3224                 }
3225               else
3226                 {
3227                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3228                                         translation_buckets,
3229                                         translation_memory_size);
3230                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3231                                                      format_session_kvp);
3232
3233                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3234                                         translation_buckets,
3235                                         translation_memory_size);
3236                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3237                                                      format_session_kvp);
3238                 }
3239
3240               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3241                                     user_memory_size);
3242               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3243                                                  format_user_kvp);
3244             }
3245           /* *INDENT-ON* */
3246
3247         }
3248       else
3249         {
3250           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3251           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3252         }
3253       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3254                             "static_mapping_by_local", static_mapping_buckets,
3255                             static_mapping_memory_size);
3256       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3257                                          format_static_mapping_kvp);
3258
3259       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3260                             "static_mapping_by_external",
3261                             static_mapping_buckets,
3262                             static_mapping_memory_size);
3263       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3264                                          format_static_mapping_kvp);
3265     }
3266
3267   return 0;
3268 }
3269
3270 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3271
3272 static void
3273 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3274                                  uword opaque,
3275                                  u32 sw_if_index,
3276                                  ip4_address_t * address,
3277                                  u32 address_length,
3278                                  u32 if_address_index, u32 is_delete)
3279 {
3280   snat_main_t *sm = &snat_main;
3281   snat_static_map_resolve_t *rp;
3282   snat_static_mapping_t *m;
3283   snat_session_key_t m_key;
3284   clib_bihash_kv_8_8_t kv, value;
3285   int i, rv;
3286   ip4_address_t l_addr;
3287
3288   for (i = 0; i < vec_len (sm->to_resolve); i++)
3289     {
3290       rp = sm->to_resolve + i;
3291       if (rp->addr_only == 0)
3292         continue;
3293       if (rp->sw_if_index == sw_if_index)
3294         goto match;
3295     }
3296
3297   return;
3298
3299 match:
3300   m_key.addr.as_u32 = address->as_u32;
3301   m_key.port = rp->addr_only ? 0 : rp->e_port;
3302   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3303   m_key.fib_index = sm->outside_fib_index;
3304   kv.key = m_key.as_u64;
3305   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3306     m = 0;
3307   else
3308     m = pool_elt_at_index (sm->static_mappings, value.value);
3309
3310   if (!is_delete)
3311     {
3312       /* Don't trip over lease renewal, static config */
3313       if (m)
3314         return;
3315     }
3316   else
3317     {
3318       if (!m)
3319         return;
3320     }
3321
3322   /* Indetity mapping? */
3323   if (rp->l_addr.as_u32 == 0)
3324     l_addr.as_u32 = address[0].as_u32;
3325   else
3326     l_addr.as_u32 = rp->l_addr.as_u32;
3327   /* Add the static mapping */
3328   rv = snat_add_static_mapping (l_addr,
3329                                 address[0],
3330                                 rp->l_port,
3331                                 rp->e_port,
3332                                 rp->vrf_id,
3333                                 rp->addr_only, ~0 /* sw_if_index */ ,
3334                                 rp->proto, !is_delete, rp->twice_nat,
3335                                 rp->out2in_only, rp->tag, rp->identity_nat);
3336   if (rv)
3337     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3338 }
3339
3340 static void
3341 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3342                                        uword opaque,
3343                                        u32 sw_if_index,
3344                                        ip4_address_t * address,
3345                                        u32 address_length,
3346                                        u32 if_address_index, u32 is_delete)
3347 {
3348   snat_main_t *sm = &snat_main;
3349   snat_static_map_resolve_t *rp;
3350   ip4_address_t l_addr;
3351   int i, j;
3352   int rv;
3353   u8 twice_nat = 0;
3354   snat_address_t *addresses = sm->addresses;
3355
3356   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3357     {
3358       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3359         goto match;
3360     }
3361
3362   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3363     {
3364       twice_nat = 1;
3365       addresses = sm->twice_nat_addresses;
3366       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3367         goto match;
3368     }
3369
3370   return;
3371
3372 match:
3373   if (!is_delete)
3374     {
3375       /* Don't trip over lease renewal, static config */
3376       for (j = 0; j < vec_len (addresses); j++)
3377         if (addresses[j].addr.as_u32 == address->as_u32)
3378           return;
3379
3380       (void) snat_add_address (sm, address, ~0, twice_nat);
3381       /* Scan static map resolution vector */
3382       for (j = 0; j < vec_len (sm->to_resolve); j++)
3383         {
3384           rp = sm->to_resolve + j;
3385           if (rp->addr_only)
3386             continue;
3387           /* On this interface? */
3388           if (rp->sw_if_index == sw_if_index)
3389             {
3390               /* Indetity mapping? */
3391               if (rp->l_addr.as_u32 == 0)
3392                 l_addr.as_u32 = address[0].as_u32;
3393               else
3394                 l_addr.as_u32 = rp->l_addr.as_u32;
3395               /* Add the static mapping */
3396               rv = snat_add_static_mapping (l_addr,
3397                                             address[0],
3398                                             rp->l_port,
3399                                             rp->e_port,
3400                                             rp->vrf_id,
3401                                             rp->addr_only,
3402                                             ~0 /* sw_if_index */ ,
3403                                             rp->proto,
3404                                             rp->is_add, rp->twice_nat,
3405                                             rp->out2in_only, rp->tag,
3406                                             rp->identity_nat);
3407               if (rv)
3408                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3409             }
3410         }
3411       return;
3412     }
3413   else
3414     {
3415       (void) snat_del_address (sm, address[0], 1, twice_nat);
3416       return;
3417     }
3418 }
3419
3420
3421 int
3422 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3423                             u8 twice_nat)
3424 {
3425   ip4_main_t *ip4_main = sm->ip4_main;
3426   ip4_address_t *first_int_addr;
3427   snat_static_map_resolve_t *rp;
3428   u32 *indices_to_delete = 0;
3429   int i, j;
3430   u32 *auto_add_sw_if_indices =
3431     twice_nat ? sm->
3432     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3433
3434   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3435     );
3436
3437   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3438     {
3439       if (auto_add_sw_if_indices[i] == sw_if_index)
3440         {
3441           if (is_del)
3442             {
3443               /* if have address remove it */
3444               if (first_int_addr)
3445                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3446               else
3447                 {
3448                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3449                     {
3450                       rp = sm->to_resolve + j;
3451                       if (rp->sw_if_index == sw_if_index)
3452                         vec_add1 (indices_to_delete, j);
3453                     }
3454                   if (vec_len (indices_to_delete))
3455                     {
3456                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3457                         vec_del1 (sm->to_resolve, j);
3458                       vec_free (indices_to_delete);
3459                     }
3460                 }
3461               if (twice_nat)
3462                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3463               else
3464                 vec_del1 (sm->auto_add_sw_if_indices, i);
3465             }
3466           else
3467             return VNET_API_ERROR_VALUE_EXIST;
3468
3469           return 0;
3470         }
3471     }
3472
3473   if (is_del)
3474     return VNET_API_ERROR_NO_SUCH_ENTRY;
3475
3476   /* add to the auto-address list */
3477   if (twice_nat)
3478     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3479   else
3480     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3481
3482   /* If the address is already bound - or static - add it now */
3483   if (first_int_addr)
3484     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3485
3486   return 0;
3487 }
3488
3489 int
3490 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3491                    snat_protocol_t proto, u32 vrf_id, int is_in)
3492 {
3493   snat_main_per_thread_data_t *tsm;
3494   clib_bihash_kv_8_8_t kv, value;
3495   ip4_header_t ip;
3496   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3497   snat_session_key_t key;
3498   snat_session_t *s;
3499   clib_bihash_8_8_t *t;
3500
3501   if (sm->endpoint_dependent)
3502     return VNET_API_ERROR_UNSUPPORTED;
3503
3504   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3505   if (sm->num_workers > 1)
3506     tsm =
3507       vec_elt_at_index (sm->per_thread_data,
3508                         sm->worker_in2out_cb (&ip, fib_index));
3509   else
3510     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3511
3512   key.addr.as_u32 = addr->as_u32;
3513   key.port = clib_host_to_net_u16 (port);
3514   key.protocol = proto;
3515   key.fib_index = fib_index;
3516   kv.key = key.as_u64;
3517   t = is_in ? &tsm->in2out : &tsm->out2in;
3518   if (!clib_bihash_search_8_8 (t, &kv, &value))
3519     {
3520       if (pool_is_free_index (tsm->sessions, value.value))
3521         return VNET_API_ERROR_UNSPECIFIED;
3522
3523       s = pool_elt_at_index (tsm->sessions, value.value);
3524       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3525       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3526       return 0;
3527     }
3528
3529   return VNET_API_ERROR_NO_SUCH_ENTRY;
3530 }
3531
3532 int
3533 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3534                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3535                       u32 vrf_id, int is_in)
3536 {
3537   ip4_header_t ip;
3538   clib_bihash_16_8_t *t;
3539   nat_ed_ses_key_t key;
3540   clib_bihash_kv_16_8_t kv, value;
3541   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3542   snat_session_t *s;
3543   snat_main_per_thread_data_t *tsm;
3544
3545   if (!sm->endpoint_dependent)
3546     return VNET_API_ERROR_FEATURE_DISABLED;
3547
3548   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3549   if (sm->num_workers > 1)
3550     tsm =
3551       vec_elt_at_index (sm->per_thread_data,
3552                         sm->worker_in2out_cb (&ip, fib_index));
3553   else
3554     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3555
3556   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3557   key.l_addr.as_u32 = addr->as_u32;
3558   key.r_addr.as_u32 = eh_addr->as_u32;
3559   key.l_port = clib_host_to_net_u16 (port);
3560   key.r_port = clib_host_to_net_u16 (eh_port);
3561   key.proto = proto;
3562   key.fib_index = fib_index;
3563   kv.key[0] = key.as_u64[0];
3564   kv.key[1] = key.as_u64[1];
3565   if (clib_bihash_search_16_8 (t, &kv, &value))
3566     return VNET_API_ERROR_NO_SUCH_ENTRY;
3567
3568   if (pool_is_free_index (tsm->sessions, value.value))
3569     return VNET_API_ERROR_UNSPECIFIED;
3570   s = pool_elt_at_index (tsm->sessions, value.value);
3571   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3572   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3573   return 0;
3574 }
3575
3576 void
3577 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3578 {
3579   snat_main_t *sm = &snat_main;
3580
3581   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3582   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3583   sm->psid = psid;
3584   sm->psid_offset = psid_offset;
3585   sm->psid_length = psid_length;
3586 }
3587
3588 void
3589 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3590 {
3591   snat_main_t *sm = &snat_main;
3592
3593   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3594   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3595   sm->start_port = start_port;
3596   sm->end_port = end_port;
3597 }
3598
3599 void
3600 nat_set_alloc_addr_and_port_default (void)
3601 {
3602   snat_main_t *sm = &snat_main;
3603
3604   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3605   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3606 }
3607
3608 /*
3609  * fd.io coding-style-patch-verification: ON
3610  *
3611  * Local Variables:
3612  * eval: (c-set-style "gnu")
3613  * End:
3614  */