src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35
  36 #include <vpp/app/version.h>
  37
  38 snat_main_t snat_main;
  39
  40 /* *INDENT-OFF* */
  41
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat44-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  47 };
  48 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat44-out2in",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-dhcp-client-detect"),
  53 };
  54 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-classify",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-in2out",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-out2in",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-det-classify",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  74 };
  75 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-ed-in2out",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-ed-out2in",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  84                                "ip4-dhcp-client-detect"),
  85 };
  86 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-ed-classify",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-in2out-worker-handoff",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  95 };
  96 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-out2in-worker-handoff",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-handoff-classify",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-in2out-fast",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 111 };
 112 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-out2in-fast",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 116                                "ip4-dhcp-client-detect"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-hairpin-dst",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ed-hairpin-dst",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 127 };
 128
 129 /* Hook up output features */
 130 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 131   .arc_name = "ip4-output",
 132   .node_name = "nat44-in2out-output",
 133   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 136   .arc_name = "ip4-output",
 137   .node_name = "nat44-in2out-output-worker-handoff",
 138   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 141   .arc_name = "ip4-output",
 142   .node_name = "nat44-hairpin-src",
 143   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 144 };
 145 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-ed-in2out-output",
 148   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 149 };
 150 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-ed-hairpin-src",
 153   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 154 };
 155
 156 /* Hook up ip4-local features */
 157 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 158 {
 159   .arc_name = "ip4-local",
 160   .node_name = "nat44-hairpinning",
 161   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 164 {
 165   .arc_name = "ip4-local",
 166   .node_name = "nat44-ed-hairpinning",
 167   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 168 };
 169
 170
 171 VLIB_PLUGIN_REGISTER () = {
 172     .version = VPP_BUILD_VER,
 173     .description = "Network Address Translation",
 174 };
 175 /* *INDENT-ON* */
 176
 177 void
 178 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 179 {
 180   snat_session_key_t key;
 181   clib_bihash_kv_8_8_t kv;
 182   nat_ed_ses_key_t ed_key;
 183   clib_bihash_kv_16_8_t ed_kv;
 184   snat_main_per_thread_data_t *tsm =
 185     vec_elt_at_index (sm->per_thread_data, thread_index);
 186
 187   if (is_fwd_bypass_session (s))
 188     {
 189       ed_key.l_addr = s->in2out.addr;
 190       ed_key.r_addr = s->ext_host_addr;
 191       ed_key.l_port = s->in2out.port;
 192       ed_key.r_port = s->ext_host_port;
 193       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 194       ed_key.fib_index = 0;
 195       ed_kv.key[0] = ed_key.as_u64[0];
 196       ed_kv.key[1] = ed_key.as_u64[1];
 197       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 198         nat_log_warn ("in2out_ed key del failed");
 199       return;
 200     }
 201
 202   /* session lookup tables */
 203   if (is_ed_session (s))
 204     {
 205       if (is_affinity_sessions (s))
 206         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 207                              s->in2out.protocol, s->out2in.port);
 208       ed_key.l_addr = s->out2in.addr;
 209       ed_key.r_addr = s->ext_host_addr;
 210       ed_key.fib_index = s->out2in.fib_index;
 211       if (snat_is_unk_proto_session (s))
 212         {
 213           ed_key.proto = s->in2out.port;
 214           ed_key.r_port = 0;
 215           ed_key.l_port = 0;
 216         }
 217       else
 218         {
 219           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 220           ed_key.l_port = s->out2in.port;
 221           ed_key.r_port = s->ext_host_port;
 222         }
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 226         nat_log_warn ("out2in_ed key del failed");
 227       ed_key.l_addr = s->in2out.addr;
 228       ed_key.fib_index = s->in2out.fib_index;
 229       if (!snat_is_unk_proto_session (s))
 230         ed_key.l_port = s->in2out.port;
 231       if (is_twice_nat_session (s))
 232         {
 233           ed_key.r_addr = s->ext_host_nat_addr;
 234           ed_key.r_port = s->ext_host_nat_port;
 235         }
 236       ed_kv.key[0] = ed_key.as_u64[0];
 237       ed_kv.key[1] = ed_key.as_u64[1];
 238       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 239         nat_log_warn ("in2out_ed key del failed");
 240
 241       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 242                              &s->in2out.addr, s->in2out.port,
 243                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 244                              &s->out2in.addr, s->out2in.port,
 245                              &s->ext_host_addr, s->ext_host_port,
 246                              s->in2out.protocol, is_twice_nat_session (s));
 247     }
 248   else
 249     {
 250       kv.key = s->in2out.as_u64;
 251       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 252         nat_log_warn ("in2out key del failed");
 253       kv.key = s->out2in.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 255         nat_log_warn ("out2in key del failed");
 256
 257       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 258                                &s->in2out.addr, s->in2out.port,
 259                                &s->out2in.addr, s->out2in.port,
 260                                s->in2out.protocol);
 261     }
 262
 263   if (snat_is_unk_proto_session (s))
 264     return;
 265
 266   /* log NAT event */
 267   snat_ipfix_logging_nat44_ses_delete (thread_index,
 268                                        s->in2out.addr.as_u32,
 269                                        s->out2in.addr.as_u32,
 270                                        s->in2out.protocol,
 271                                        s->in2out.port,
 272                                        s->out2in.port, s->in2out.fib_index);
 273
 274   /* Twice NAT address and port for external host */
 275   if (is_twice_nat_session (s))
 276     {
 277       key.protocol = s->in2out.protocol;
 278       key.port = s->ext_host_nat_port;
 279       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 280       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 281                                           thread_index, &key);
 282     }
 283
 284   if (snat_is_session_static (s))
 285     return;
 286
 287   snat_free_outside_address_and_port (sm->addresses, thread_index,
 288                                       &s->out2in);
 289 }
 290
 291 snat_user_t *
 292 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 293                         u32 thread_index)
 294 {
 295   snat_user_t *u = 0;
 296   snat_user_key_t user_key;
 297   clib_bihash_kv_8_8_t kv, value;
 298   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 299   dlist_elt_t *per_user_list_head_elt;
 300
 301   user_key.addr.as_u32 = addr->as_u32;
 302   user_key.fib_index = fib_index;
 303   kv.key = user_key.as_u64;
 304
 305   /* Ever heard of the "user" = src ip4 address before? */
 306   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 307     {
 308       /* no, make a new one */
 309       pool_get (tsm->users, u);
 310       clib_memset (u, 0, sizeof (*u));
 311       u->addr.as_u32 = addr->as_u32;
 312       u->fib_index = fib_index;
 313
 314       pool_get (tsm->list_pool, per_user_list_head_elt);
 315
 316       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 317         tsm->list_pool;
 318
 319       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 320
 321       kv.value = u - tsm->users;
 322
 323       /* add user */
 324       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 325         nat_log_warn ("user_hash keay add failed");
 326
 327       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 328                                pool_elts (tsm->users));
 329     }
 330   else
 331     {
 332       u = pool_elt_at_index (tsm->users, value.value);
 333     }
 334
 335   return u;
 336 }
 337
 338 snat_session_t *
 339 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 340                               u32 thread_index)
 341 {
 342   snat_session_t *s;
 343   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 344   u32 oldest_per_user_translation_list_index, session_index;
 345   dlist_elt_t *oldest_per_user_translation_list_elt;
 346   dlist_elt_t *per_user_translation_list_elt;
 347
 348   /* Over quota? Recycle the least recently used translation */
 349   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 350     {
 351       oldest_per_user_translation_list_index =
 352         clib_dlist_remove_head (tsm->list_pool,
 353                                 u->sessions_per_user_list_head_index);
 354
 355       ASSERT (oldest_per_user_translation_list_index != ~0);
 356
 357       /* Add it back to the end of the LRU list */
 358       clib_dlist_addtail (tsm->list_pool,
 359                           u->sessions_per_user_list_head_index,
 360                           oldest_per_user_translation_list_index);
 361       /* Get the list element */
 362       oldest_per_user_translation_list_elt =
 363         pool_elt_at_index (tsm->list_pool,
 364                            oldest_per_user_translation_list_index);
 365
 366       /* Get the session index from the list element */
 367       session_index = oldest_per_user_translation_list_elt->value;
 368
 369       /* Get the session */
 370       s = pool_elt_at_index (tsm->sessions, session_index);
 371       nat_free_session_data (sm, s, thread_index);
 372       if (snat_is_session_static (s))
 373         u->nstaticsessions--;
 374       else
 375         u->nsessions--;
 376       s->flags = 0;
 377       s->total_bytes = 0;
 378       s->total_pkts = 0;
 379       s->state = 0;
 380       s->ext_host_addr.as_u32 = 0;
 381       s->ext_host_port = 0;
 382       s->ext_host_nat_addr.as_u32 = 0;
 383       s->ext_host_nat_port = 0;
 384     }
 385   else
 386     {
 387       pool_get (tsm->sessions, s);
 388       clib_memset (s, 0, sizeof (*s));
 389
 390       /* Create list elts */
 391       pool_get (tsm->list_pool, per_user_translation_list_elt);
 392       clib_dlist_init (tsm->list_pool,
 393                        per_user_translation_list_elt - tsm->list_pool);
 394
 395       per_user_translation_list_elt->value = s - tsm->sessions;
 396       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 397       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 398
 399       clib_dlist_addtail (tsm->list_pool,
 400                           s->per_user_list_head_index,
 401                           per_user_translation_list_elt - tsm->list_pool);
 402
 403       s->user_index = u - tsm->users;
 404       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 405                                pool_elts (tsm->sessions));
 406     }
 407
 408   return s;
 409 }
 410
 411 snat_session_t *
 412 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 413                       f64 now)
 414 {
 415   snat_session_t *s;
 416   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 417   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 418   u32 oldest_index;
 419   u64 sess_timeout_time;
 420
 421   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 422     goto alloc_new;
 423
 424   oldest_index =
 425     clib_dlist_remove_head (tsm->list_pool,
 426                             u->sessions_per_user_list_head_index);
 427   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 428   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 429   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 430   if (now >= sess_timeout_time)
 431     {
 432       clib_dlist_addtail (tsm->list_pool,
 433                           u->sessions_per_user_list_head_index, oldest_index);
 434       nat_free_session_data (sm, s, thread_index);
 435       if (snat_is_session_static (s))
 436         u->nstaticsessions--;
 437       else
 438         u->nsessions--;
 439       s->flags = 0;
 440       s->total_bytes = 0;
 441       s->total_pkts = 0;
 442       s->state = 0;
 443       s->ext_host_addr.as_u32 = 0;
 444       s->ext_host_port = 0;
 445       s->ext_host_nat_addr.as_u32 = 0;
 446       s->ext_host_nat_port = 0;
 447     }
 448   else
 449     {
 450       clib_dlist_addhead (tsm->list_pool,
 451                           u->sessions_per_user_list_head_index, oldest_index);
 452       if ((u->nsessions + u->nstaticsessions) >=
 453           sm->max_translations_per_user)
 454         {
 455           nat_log_warn ("max translations per user %U", format_ip4_address,
 456                         &u->addr);
 457           snat_ipfix_logging_max_entries_per_user
 458             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 459           return 0;
 460         }
 461       else
 462         {
 463         alloc_new:
 464           pool_get (tsm->sessions, s);
 465           clib_memset (s, 0, sizeof (*s));
 466
 467           /* Create list elts */
 468           pool_get (tsm->list_pool, per_user_translation_list_elt);
 469           clib_dlist_init (tsm->list_pool,
 470                            per_user_translation_list_elt - tsm->list_pool);
 471
 472           per_user_translation_list_elt->value = s - tsm->sessions;
 473           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 474           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 475
 476           clib_dlist_addtail (tsm->list_pool,
 477                               s->per_user_list_head_index,
 478                               per_user_translation_list_elt - tsm->list_pool);
 479         }
 480
 481       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 482                                pool_elts (tsm->sessions));
 483     }
 484
 485   return s;
 486 }
 487
 488 void
 489 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 490                           int is_add)
 491 {
 492   fib_prefix_t prefix = {
 493     .fp_len = p_len,
 494     .fp_proto = FIB_PROTOCOL_IP4,
 495     .fp_addr = {
 496                 .ip4.as_u32 = addr->as_u32,
 497                 },
 498   };
 499   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 500
 501   if (is_add)
 502     fib_table_entry_update_one_path (fib_index,
 503                                      &prefix,
 504                                      FIB_SOURCE_PLUGIN_LOW,
 505                                      (FIB_ENTRY_FLAG_CONNECTED |
 506                                       FIB_ENTRY_FLAG_LOCAL |
 507                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 508                                      DPO_PROTO_IP4,
 509                                      NULL,
 510                                      sw_if_index,
 511                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 512   else
 513     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 514 }
 515
 516 int
 517 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 518                   u8 twice_nat)
 519 {
 520   snat_address_t *ap;
 521   snat_interface_t *i;
 522   vlib_thread_main_t *tm = vlib_get_thread_main ();
 523
 524   if (twice_nat && !sm->endpoint_dependent)
 525     return VNET_API_ERROR_FEATURE_DISABLED;
 526
 527   /* Check if address already exists */
 528   /* *INDENT-OFF* */
 529   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 530     {
 531       if (ap->addr.as_u32 == addr->as_u32)
 532         return VNET_API_ERROR_VALUE_EXIST;
 533     }
 534   /* *INDENT-ON* */
 535
 536   if (twice_nat)
 537     vec_add2 (sm->twice_nat_addresses, ap, 1);
 538   else
 539     vec_add2 (sm->addresses, ap, 1);
 540
 541   ap->addr = *addr;
 542   if (vrf_id != ~0)
 543     ap->fib_index =
 544       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 545                                          FIB_SOURCE_PLUGIN_LOW);
 546   else
 547     ap->fib_index = ~0;
 548 #define _(N, i, n, s) \
 549   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 550   ap->busy_##n##_ports = 0; \
 551   ap->busy_##n##_ports_per_thread = 0;\
 552   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 553   foreach_snat_protocol
 554 #undef _
 555     if (twice_nat)
 556     return 0;
 557
 558   /* Add external address to FIB */
 559   /* *INDENT-OFF* */
 560   pool_foreach (i, sm->interfaces,
 561   ({
 562     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 563       continue;
 564
 565     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 566     break;
 567   }));
 568   pool_foreach (i, sm->output_feature_interfaces,
 569   ({
 570     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 571       continue;
 572
 573     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 574     break;
 575   }));
 576   /* *INDENT-ON* */
 577
 578   return 0;
 579 }
 580
 581 static int
 582 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 583 {
 584   snat_static_mapping_t *m;
 585   /* *INDENT-OFF* */
 586   pool_foreach (m, sm->static_mappings,
 587   ({
 588       if (is_addr_only_static_mapping (m) ||
 589           is_out2in_only_static_mapping (m) ||
 590           is_identity_static_mapping (m))
 591         continue;
 592       if (m->external_addr.as_u32 == addr.as_u32)
 593         return 1;
 594   }));
 595   /* *INDENT-ON* */
 596
 597   return 0;
 598 }
 599
 600 void
 601 increment_v4_address (ip4_address_t * a)
 602 {
 603   u32 v;
 604
 605   v = clib_net_to_host_u32 (a->as_u32) + 1;
 606   a->as_u32 = clib_host_to_net_u32 (v);
 607 }
 608
 609 static void
 610 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 611                                        ip4_address_t l_addr,
 612                                        u16 l_port,
 613                                        u32 sw_if_index,
 614                                        u16 e_port,
 615                                        u32 vrf_id,
 616                                        snat_protocol_t proto,
 617                                        int addr_only, int is_add, u8 * tag,
 618                                        int twice_nat, int out2in_only,
 619                                        int identity_nat)
 620 {
 621   snat_static_map_resolve_t *rp;
 622
 623   vec_add2 (sm->to_resolve, rp, 1);
 624   rp->l_addr.as_u32 = l_addr.as_u32;
 625   rp->l_port = l_port;
 626   rp->sw_if_index = sw_if_index;
 627   rp->e_port = e_port;
 628   rp->vrf_id = vrf_id;
 629   rp->proto = proto;
 630   rp->addr_only = addr_only;
 631   rp->is_add = is_add;
 632   rp->twice_nat = twice_nat;
 633   rp->out2in_only = out2in_only;
 634   rp->identity_nat = identity_nat;
 635   rp->tag = vec_dup (tag);
 636 }
 637
 638 static u32
 639 get_thread_idx_by_port (u16 e_port)
 640 {
 641   snat_main_t *sm = &snat_main;
 642   u32 thread_idx = sm->num_workers;
 643   if (sm->num_workers > 1)
 644     {
 645       thread_idx =
 646         sm->first_worker_index +
 647         sm->workers[(e_port - 1024) / sm->port_per_thread];
 648     }
 649   return thread_idx;
 650 }
 651
 652 int
 653 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 654                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 655                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 656                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 657                          u8 identity_nat)
 658 {
 659   snat_main_t *sm = &snat_main;
 660   snat_static_mapping_t *m;
 661   snat_session_key_t m_key;
 662   clib_bihash_kv_8_8_t kv, value;
 663   snat_address_t *a = 0;
 664   u32 fib_index = ~0;
 665   snat_interface_t *interface;
 666   int i;
 667   snat_main_per_thread_data_t *tsm;
 668   snat_user_key_t u_key;
 669   snat_user_t *u;
 670   dlist_elt_t *head, *elt;
 671   u32 elt_index, head_index;
 672   u32 ses_index;
 673   u64 user_index;
 674   snat_session_t *s;
 675   snat_static_map_resolve_t *rp, *rp_match = 0;
 676   nat44_lb_addr_port_t *local;
 677   u32 find = ~0;
 678
 679   if (!sm->endpoint_dependent)
 680     {
 681       if (twice_nat || out2in_only)
 682         return VNET_API_ERROR_FEATURE_DISABLED;
 683     }
 684
 685   /* If the external address is a specific interface address */
 686   if (sw_if_index != ~0)
 687     {
 688       ip4_address_t *first_int_addr;
 689
 690       for (i = 0; i < vec_len (sm->to_resolve); i++)
 691         {
 692           rp = sm->to_resolve + i;
 693           if (rp->sw_if_index != sw_if_index ||
 694               rp->l_addr.as_u32 != l_addr.as_u32 ||
 695               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 696             continue;
 697
 698           if (!addr_only)
 699             {
 700               if (rp->l_port != l_port || rp->e_port != e_port
 701                   || rp->proto != proto)
 702                 continue;
 703             }
 704
 705           rp_match = rp;
 706           break;
 707         }
 708
 709       /* Might be already set... */
 710       first_int_addr = ip4_interface_first_address
 711         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 712
 713       if (is_add)
 714         {
 715           if (rp_match)
 716             return VNET_API_ERROR_VALUE_EXIST;
 717
 718           snat_add_static_mapping_when_resolved
 719             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 720              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 721
 722           /* DHCP resolution required? */
 723           if (first_int_addr == 0)
 724             {
 725               return 0;
 726             }
 727           else
 728             {
 729               e_addr.as_u32 = first_int_addr->as_u32;
 730               /* Identity mapping? */
 731               if (l_addr.as_u32 == 0)
 732                 l_addr.as_u32 = e_addr.as_u32;
 733             }
 734         }
 735       else
 736         {
 737           if (!rp_match)
 738             return VNET_API_ERROR_NO_SUCH_ENTRY;
 739
 740           vec_del1 (sm->to_resolve, i);
 741
 742           if (first_int_addr)
 743             {
 744               e_addr.as_u32 = first_int_addr->as_u32;
 745               /* Identity mapping? */
 746               if (l_addr.as_u32 == 0)
 747                 l_addr.as_u32 = e_addr.as_u32;
 748             }
 749           else
 750             return 0;
 751         }
 752     }
 753
 754   m_key.addr = e_addr;
 755   m_key.port = addr_only ? 0 : e_port;
 756   m_key.protocol = addr_only ? 0 : proto;
 757   m_key.fib_index = 0;
 758   kv.key = m_key.as_u64;
 759   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 760     m = 0;
 761   else
 762     m = pool_elt_at_index (sm->static_mappings, value.value);
 763
 764   if (is_add)
 765     {
 766       if (m)
 767         {
 768           if (is_identity_static_mapping (m))
 769             {
 770               /* *INDENT-OFF* */
 771               pool_foreach (local, m->locals,
 772               ({
 773                 if (local->vrf_id == vrf_id)
 774                   return VNET_API_ERROR_VALUE_EXIST;
 775               }));
 776               /* *INDENT-ON* */
 777               pool_get (m->locals, local);
 778               local->vrf_id = vrf_id;
 779               local->fib_index =
 780                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 781                                                    FIB_SOURCE_PLUGIN_LOW);
 782               m_key.addr = m->local_addr;
 783               m_key.port = m->local_port;
 784               m_key.protocol = m->proto;
 785               m_key.fib_index = local->fib_index;
 786               kv.key = m_key.as_u64;
 787               kv.value = m - sm->static_mappings;
 788               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 789               return 0;
 790             }
 791           else
 792             return VNET_API_ERROR_VALUE_EXIST;
 793         }
 794
 795       if (twice_nat && addr_only)
 796         return VNET_API_ERROR_UNSUPPORTED;
 797
 798       /* Convert VRF id to FIB index */
 799       if (vrf_id != ~0)
 800         fib_index =
 801           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 802                                              FIB_SOURCE_PLUGIN_LOW);
 803       /* If not specified use inside VRF id from SNAT plugin startup config */
 804       else
 805         {
 806           fib_index = sm->inside_fib_index;
 807           vrf_id = sm->inside_vrf_id;
 808         }
 809
 810       if (!(out2in_only || identity_nat))
 811         {
 812           m_key.addr = l_addr;
 813           m_key.port = addr_only ? 0 : l_port;
 814           m_key.protocol = addr_only ? 0 : proto;
 815           m_key.fib_index = fib_index;
 816           kv.key = m_key.as_u64;
 817           if (!clib_bihash_search_8_8
 818               (&sm->static_mapping_by_local, &kv, &value))
 819             return VNET_API_ERROR_VALUE_EXIST;
 820         }
 821
 822       /* Find external address in allocated addresses and reserve port for
 823          address and port pair mapping when dynamic translations enabled */
 824       if (!(addr_only || sm->static_mapping_only || out2in_only))
 825         {
 826           for (i = 0; i < vec_len (sm->addresses); i++)
 827             {
 828               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 829                 {
 830                   a = sm->addresses + i;
 831                   /* External port must be unused */
 832                   switch (proto)
 833                     {
 834 #define _(N, j, n, s) \
 835                     case SNAT_PROTOCOL_##N: \
 836                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 837                         return VNET_API_ERROR_INVALID_VALUE; \
 838                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 839                       if (e_port > 1024) \
 840                         { \
 841                           a->busy_##n##_ports++; \
 842                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 843                         } \
 844                       break;
 845                       foreach_snat_protocol
 846 #undef _
 847                     default:
 848                       nat_log_info ("unknown protocol");
 849                       return VNET_API_ERROR_INVALID_VALUE_2;
 850                     }
 851                   break;
 852                 }
 853             }
 854           /* External address must be allocated */
 855           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 856             {
 857               if (sw_if_index != ~0)
 858                 {
 859                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 860                     {
 861                       rp = sm->to_resolve + i;
 862                       if (rp->addr_only)
 863                         continue;
 864                       if (rp->sw_if_index != sw_if_index &&
 865                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 866                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 867                           rp->e_port != e_port && rp->proto != proto)
 868                         continue;
 869
 870                       vec_del1 (sm->to_resolve, i);
 871                       break;
 872                     }
 873                 }
 874               return VNET_API_ERROR_NO_SUCH_ENTRY;
 875             }
 876         }
 877
 878       pool_get (sm->static_mappings, m);
 879       clib_memset (m, 0, sizeof (*m));
 880       m->tag = vec_dup (tag);
 881       m->local_addr = l_addr;
 882       m->external_addr = e_addr;
 883       m->twice_nat = twice_nat;
 884       if (out2in_only)
 885         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 886       if (addr_only)
 887         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 888       if (identity_nat)
 889         {
 890           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 891           pool_get (m->locals, local);
 892           local->vrf_id = vrf_id;
 893           local->fib_index = fib_index;
 894         }
 895       else
 896         {
 897           m->vrf_id = vrf_id;
 898           m->fib_index = fib_index;
 899         }
 900       if (!addr_only)
 901         {
 902           m->local_port = l_port;
 903           m->external_port = e_port;
 904           m->proto = proto;
 905         }
 906
 907       if (sm->num_workers > 1)
 908         {
 909           ip4_header_t ip = {
 910             .src_address = m->local_addr,
 911           };
 912           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 913           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 914         }
 915       else
 916         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 917
 918       m_key.addr = m->local_addr;
 919       m_key.port = m->local_port;
 920       m_key.protocol = m->proto;
 921       m_key.fib_index = fib_index;
 922       kv.key = m_key.as_u64;
 923       kv.value = m - sm->static_mappings;
 924       if (!out2in_only)
 925         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 926
 927       m_key.addr = m->external_addr;
 928       m_key.port = m->external_port;
 929       m_key.fib_index = 0;
 930       kv.key = m_key.as_u64;
 931       kv.value = m - sm->static_mappings;
 932       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 933
 934       /* Delete dynamic sessions matching local address (+ local port) */
 935       if (!(sm->static_mapping_only))
 936         {
 937           u_key.addr = m->local_addr;
 938           u_key.fib_index = m->fib_index;
 939           kv.key = u_key.as_u64;
 940           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 941             {
 942               user_index = value.value;
 943               u = pool_elt_at_index (tsm->users, user_index);
 944               if (u->nsessions)
 945                 {
 946                   head_index = u->sessions_per_user_list_head_index;
 947                   head = pool_elt_at_index (tsm->list_pool, head_index);
 948                   elt_index = head->next;
 949                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 950                   ses_index = elt->value;
 951                   while (ses_index != ~0)
 952                     {
 953                       s = pool_elt_at_index (tsm->sessions, ses_index);
 954                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 955                       ses_index = elt->value;
 956
 957                       if (snat_is_session_static (s))
 958                         continue;
 959
 960                       if (!addr_only
 961                           && (clib_net_to_host_u16 (s->in2out.port) !=
 962                               m->local_port))
 963                         continue;
 964
 965                       nat_free_session_data (sm, s,
 966                                              tsm - sm->per_thread_data);
 967                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 968
 969                       if (!addr_only && !sm->endpoint_dependent)
 970                         break;
 971                     }
 972                 }
 973             }
 974         }
 975     }
 976   else
 977     {
 978       if (!m)
 979         {
 980           if (sw_if_index != ~0)
 981             return 0;
 982           else
 983             return VNET_API_ERROR_NO_SUCH_ENTRY;
 984         }
 985
 986       if (identity_nat)
 987         {
 988           if (vrf_id == ~0)
 989             vrf_id = sm->inside_vrf_id;
 990
 991           /* *INDENT-OFF* */
 992           pool_foreach (local, m->locals,
 993           ({
 994             if (local->vrf_id == vrf_id)
 995               find = local - m->locals;
 996           }));
 997           /* *INDENT-ON* */
 998           if (find == ~0)
 999             return VNET_API_ERROR_NO_SUCH_ENTRY;
1000
1001           local = pool_elt_at_index (m->locals, find);
1002           fib_index = local->fib_index;
1003           pool_put (m->locals, local);
1004         }
1005       else
1006         fib_index = m->fib_index;
1007
1008       /* Free external address port */
1009       if (!(addr_only || sm->static_mapping_only || out2in_only))
1010         {
1011           for (i = 0; i < vec_len (sm->addresses); i++)
1012             {
1013               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1014                 {
1015                   a = sm->addresses + i;
1016                   switch (proto)
1017                     {
1018 #define _(N, j, n, s) \
1019                     case SNAT_PROTOCOL_##N: \
1020                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1021                       if (e_port > 1024) \
1022                         { \
1023                           a->busy_##n##_ports--; \
1024                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1025                         } \
1026                       break;
1027                       foreach_snat_protocol
1028 #undef _
1029                     default:
1030                       nat_log_info ("unknown protocol");
1031                       return VNET_API_ERROR_INVALID_VALUE_2;
1032                     }
1033                   break;
1034                 }
1035             }
1036         }
1037
1038       if (sm->num_workers > 1)
1039         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1040       else
1041         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1042
1043       m_key.addr = m->local_addr;
1044       m_key.port = m->local_port;
1045       m_key.protocol = m->proto;
1046       m_key.fib_index = fib_index;
1047       kv.key = m_key.as_u64;
1048       if (!out2in_only)
1049         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1050
1051       /* Delete session(s) for static mapping if exist */
1052       if (!(sm->static_mapping_only) ||
1053           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1054         {
1055           u_key.addr = m->local_addr;
1056           u_key.fib_index = fib_index;
1057           kv.key = u_key.as_u64;
1058           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1059             {
1060               user_index = value.value;
1061               u = pool_elt_at_index (tsm->users, user_index);
1062               if (u->nstaticsessions)
1063                 {
1064                   head_index = u->sessions_per_user_list_head_index;
1065                   head = pool_elt_at_index (tsm->list_pool, head_index);
1066                   elt_index = head->next;
1067                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1068                   ses_index = elt->value;
1069                   while (ses_index != ~0)
1070                     {
1071                       s = pool_elt_at_index (tsm->sessions, ses_index);
1072                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1073                       ses_index = elt->value;
1074
1075                       if (!addr_only)
1076                         {
1077                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1078                               (clib_net_to_host_u16 (s->out2in.port) !=
1079                                e_port))
1080                             continue;
1081                         }
1082
1083                       if (is_lb_session (s))
1084                         continue;
1085
1086                       if (!snat_is_session_static (s))
1087                         continue;
1088
1089                       nat_free_session_data (sm, s,
1090                                              tsm - sm->per_thread_data);
1091                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1092
1093                       if (!addr_only && !sm->endpoint_dependent)
1094                         break;
1095                     }
1096                 }
1097             }
1098         }
1099
1100       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1101       if (pool_elts (m->locals))
1102         return 0;
1103
1104       m_key.addr = m->external_addr;
1105       m_key.port = m->external_port;
1106       m_key.fib_index = 0;
1107       kv.key = m_key.as_u64;
1108       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1109
1110       vec_free (m->tag);
1111       vec_free (m->workers);
1112       /* Delete static mapping from pool */
1113       pool_put (sm->static_mappings, m);
1114     }
1115
1116   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1117     return 0;
1118
1119   /* Add/delete external address to FIB */
1120   /* *INDENT-OFF* */
1121   pool_foreach (interface, sm->interfaces,
1122   ({
1123     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1124       continue;
1125
1126     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1127     break;
1128   }));
1129   pool_foreach (interface, sm->output_feature_interfaces,
1130   ({
1131     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1132       continue;
1133
1134     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1135     break;
1136   }));
1137   /* *INDENT-ON* */
1138
1139   return 0;
1140 }
1141
1142 int
1143 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1144                                  snat_protocol_t proto,
1145                                  nat44_lb_addr_port_t * locals, u8 is_add,
1146                                  twice_nat_type_t twice_nat, u8 out2in_only,
1147                                  u8 * tag, u32 affinity)
1148 {
1149   snat_main_t *sm = &snat_main;
1150   snat_static_mapping_t *m;
1151   snat_session_key_t m_key;
1152   clib_bihash_kv_8_8_t kv, value;
1153   snat_address_t *a = 0;
1154   int i;
1155   nat44_lb_addr_port_t *local;
1156   u32 elt_index, head_index, ses_index;
1157   snat_main_per_thread_data_t *tsm;
1158   snat_user_key_t u_key;
1159   snat_user_t *u;
1160   snat_session_t *s;
1161   dlist_elt_t *head, *elt;
1162   uword *bitmap = 0;
1163
1164   if (!sm->endpoint_dependent)
1165     return VNET_API_ERROR_FEATURE_DISABLED;
1166
1167   m_key.addr = e_addr;
1168   m_key.port = e_port;
1169   m_key.protocol = proto;
1170   m_key.fib_index = 0;
1171   kv.key = m_key.as_u64;
1172   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1173     m = 0;
1174   else
1175     m = pool_elt_at_index (sm->static_mappings, value.value);
1176
1177   if (is_add)
1178     {
1179       if (m)
1180         return VNET_API_ERROR_VALUE_EXIST;
1181
1182       if (vec_len (locals) < 2)
1183         return VNET_API_ERROR_INVALID_VALUE;
1184
1185       /* Find external address in allocated addresses and reserve port for
1186          address and port pair mapping when dynamic translations enabled */
1187       if (!(sm->static_mapping_only || out2in_only))
1188         {
1189           for (i = 0; i < vec_len (sm->addresses); i++)
1190             {
1191               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1192                 {
1193                   a = sm->addresses + i;
1194                   /* External port must be unused */
1195                   switch (proto)
1196                     {
1197 #define _(N, j, n, s) \
1198                     case SNAT_PROTOCOL_##N: \
1199                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1200                         return VNET_API_ERROR_INVALID_VALUE; \
1201                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1202                       if (e_port > 1024) \
1203                         { \
1204                           a->busy_##n##_ports++; \
1205                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1206                         } \
1207                       break;
1208                       foreach_snat_protocol
1209 #undef _
1210                     default:
1211                       nat_log_info ("unknown protocol");
1212                       return VNET_API_ERROR_INVALID_VALUE_2;
1213                     }
1214                   break;
1215                 }
1216             }
1217           /* External address must be allocated */
1218           if (!a)
1219             return VNET_API_ERROR_NO_SUCH_ENTRY;
1220         }
1221
1222       pool_get (sm->static_mappings, m);
1223       clib_memset (m, 0, sizeof (*m));
1224       m->tag = vec_dup (tag);
1225       m->external_addr = e_addr;
1226       m->external_port = e_port;
1227       m->proto = proto;
1228       m->twice_nat = twice_nat;
1229       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1230       if (out2in_only)
1231         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1232       m->affinity = affinity;
1233
1234       if (affinity)
1235         m->affinity_per_service_list_head_index =
1236           nat_affinity_get_per_service_list_head_index ();
1237       else
1238         m->affinity_per_service_list_head_index = ~0;
1239
1240       m_key.addr = m->external_addr;
1241       m_key.port = m->external_port;
1242       m_key.protocol = m->proto;
1243       m_key.fib_index = 0;
1244       kv.key = m_key.as_u64;
1245       kv.value = m - sm->static_mappings;
1246       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1247         {
1248           nat_log_err ("static_mapping_by_external key add failed");
1249           return VNET_API_ERROR_UNSPECIFIED;
1250         }
1251
1252       m_key.fib_index = m->fib_index;
1253       for (i = 0; i < vec_len (locals); i++)
1254         {
1255           locals[i].fib_index =
1256             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1257                                                locals[i].vrf_id,
1258                                                FIB_SOURCE_PLUGIN_LOW);
1259           m_key.addr = locals[i].addr;
1260           m_key.fib_index = locals[i].fib_index;
1261           if (!out2in_only)
1262             {
1263               m_key.port = locals[i].port;
1264               kv.key = m_key.as_u64;
1265               kv.value = m - sm->static_mappings;
1266               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1267             }
1268           locals[i].prefix = (i == 0) ? locals[i].probability :
1269             (locals[i - 1].prefix + locals[i].probability);
1270           pool_get (m->locals, local);
1271           *local = locals[i];
1272           if (sm->num_workers > 1)
1273             {
1274               ip4_header_t ip = {
1275                 .src_address = locals[i].addr,
1276               };
1277               bitmap =
1278                 clib_bitmap_set (bitmap,
1279                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1280             }
1281         }
1282
1283       /* Assign workers */
1284       if (sm->num_workers > 1)
1285         {
1286           /* *INDENT-OFF* */
1287           clib_bitmap_foreach (i, bitmap,
1288             ({
1289                vec_add1(m->workers, i);
1290             }));
1291           /* *INDENT-ON* */
1292         }
1293     }
1294   else
1295     {
1296       if (!m)
1297         return VNET_API_ERROR_NO_SUCH_ENTRY;
1298
1299       if (!is_lb_static_mapping (m))
1300         return VNET_API_ERROR_INVALID_VALUE;
1301
1302       /* Free external address port */
1303       if (!(sm->static_mapping_only || out2in_only))
1304         {
1305           for (i = 0; i < vec_len (sm->addresses); i++)
1306             {
1307               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1308                 {
1309                   a = sm->addresses + i;
1310                   switch (proto)
1311                     {
1312 #define _(N, j, n, s) \
1313                     case SNAT_PROTOCOL_##N: \
1314                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1315                       if (e_port > 1024) \
1316                         { \
1317                           a->busy_##n##_ports--; \
1318                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1319                         } \
1320                       break;
1321                       foreach_snat_protocol
1322 #undef _
1323                     default:
1324                       nat_log_info ("unknown protocol");
1325                       return VNET_API_ERROR_INVALID_VALUE_2;
1326                     }
1327                   break;
1328                 }
1329             }
1330         }
1331
1332       m_key.addr = m->external_addr;
1333       m_key.port = m->external_port;
1334       m_key.protocol = m->proto;
1335       m_key.fib_index = 0;
1336       kv.key = m_key.as_u64;
1337       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1338         {
1339           nat_log_err ("static_mapping_by_external key del failed");
1340           return VNET_API_ERROR_UNSPECIFIED;
1341         }
1342
1343       /* *INDENT-OFF* */
1344       pool_foreach (local, m->locals,
1345       ({
1346           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1347                             FIB_SOURCE_PLUGIN_LOW);
1348           m_key.addr = local->addr;
1349           if (!out2in_only)
1350             {
1351               m_key.port = local->port;
1352               m_key.fib_index = local->fib_index;
1353               kv.key = m_key.as_u64;
1354               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1355                 {
1356                   nat_log_err ("static_mapping_by_local key del failed");
1357                   return VNET_API_ERROR_UNSPECIFIED;
1358                 }
1359             }
1360
1361           if (sm->num_workers > 1)
1362             {
1363               ip4_header_t ip = {
1364                 .src_address = local->addr,
1365               };
1366               tsm = vec_elt_at_index (sm->per_thread_data,
1367                                       sm->worker_in2out_cb (&ip, m->fib_index));
1368             }
1369           else
1370             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1371
1372           /* Delete sessions */
1373           u_key.addr = local->addr;
1374           u_key.fib_index = local->fib_index;
1375           kv.key = u_key.as_u64;
1376           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1377             {
1378               u = pool_elt_at_index (tsm->users, value.value);
1379               if (u->nstaticsessions)
1380                 {
1381                   head_index = u->sessions_per_user_list_head_index;
1382                   head = pool_elt_at_index (tsm->list_pool, head_index);
1383                   elt_index = head->next;
1384                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1385                   ses_index = elt->value;
1386                   while (ses_index != ~0)
1387                     {
1388                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1389                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1390                       ses_index = elt->value;
1391
1392                       if (!(is_lb_session (s)))
1393                         continue;
1394
1395                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1396                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1397                         continue;
1398
1399                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1400                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1401                     }
1402                 }
1403             }
1404       }));
1405       /* *INDENT-ON* */
1406       if (m->affinity)
1407         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1408       pool_free (m->locals);
1409       vec_free (m->tag);
1410       vec_free (m->workers);
1411
1412       pool_put (sm->static_mappings, m);
1413     }
1414
1415   return 0;
1416 }
1417
1418 int
1419 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1420                                        ip4_address_t l_addr, u16 l_port,
1421                                        snat_protocol_t proto, u32 vrf_id,
1422                                        u8 probability, u8 is_add)
1423 {
1424   snat_main_t *sm = &snat_main;
1425   snat_static_mapping_t *m = 0;
1426   snat_session_key_t m_key;
1427   clib_bihash_kv_8_8_t kv, value;
1428   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1429   snat_main_per_thread_data_t *tsm;
1430   snat_user_key_t u_key;
1431   snat_user_t *u;
1432   snat_session_t *s;
1433   dlist_elt_t *head, *elt;
1434   u32 elt_index, head_index, ses_index, *locals = 0;
1435   uword *bitmap = 0;
1436   int i;
1437
1438   if (!sm->endpoint_dependent)
1439     return VNET_API_ERROR_FEATURE_DISABLED;
1440
1441   m_key.addr = e_addr;
1442   m_key.port = e_port;
1443   m_key.protocol = proto;
1444   m_key.fib_index = 0;
1445   kv.key = m_key.as_u64;
1446   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1447     m = pool_elt_at_index (sm->static_mappings, value.value);
1448
1449   if (!m)
1450     return VNET_API_ERROR_NO_SUCH_ENTRY;
1451
1452   if (!is_lb_static_mapping (m))
1453     return VNET_API_ERROR_INVALID_VALUE;
1454
1455   /* *INDENT-OFF* */
1456   pool_foreach (local, m->locals,
1457   ({
1458     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1459         (local->vrf_id == vrf_id))
1460       {
1461         match_local = local;
1462         break;
1463       }
1464   }));
1465   /* *INDENT-ON* */
1466
1467   if (is_add)
1468     {
1469       if (match_local)
1470         return VNET_API_ERROR_VALUE_EXIST;
1471
1472       pool_get (m->locals, local);
1473       clib_memset (local, 0, sizeof (*local));
1474       local->addr.as_u32 = l_addr.as_u32;
1475       local->port = l_port;
1476       local->probability = probability;
1477       local->vrf_id = vrf_id;
1478       local->fib_index =
1479         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1480                                            FIB_SOURCE_PLUGIN_LOW);
1481
1482       if (!is_out2in_only_static_mapping (m))
1483         {
1484           m_key.addr = l_addr;
1485           m_key.port = l_port;
1486           m_key.fib_index = local->fib_index;
1487           kv.key = m_key.as_u64;
1488           kv.value = m - sm->static_mappings;
1489           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1490             nat_log_err ("static_mapping_by_local key add failed");
1491         }
1492     }
1493   else
1494     {
1495       if (!match_local)
1496         return VNET_API_ERROR_NO_SUCH_ENTRY;
1497
1498       if (pool_elts (m->locals) < 3)
1499         return VNET_API_ERROR_UNSPECIFIED;
1500
1501       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1502                         FIB_SOURCE_PLUGIN_LOW);
1503
1504       if (!is_out2in_only_static_mapping (m))
1505         {
1506           m_key.addr = l_addr;
1507           m_key.port = l_port;
1508           m_key.fib_index = match_local->fib_index;
1509           kv.key = m_key.as_u64;
1510           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1511             nat_log_err ("static_mapping_by_local key del failed");
1512         }
1513
1514       if (sm->num_workers > 1)
1515         {
1516           ip4_header_t ip = {
1517             .src_address = local->addr,
1518           };
1519           tsm = vec_elt_at_index (sm->per_thread_data,
1520                                   sm->worker_in2out_cb (&ip, m->fib_index));
1521         }
1522       else
1523         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1524
1525       /* Delete sessions */
1526       u_key.addr = match_local->addr;
1527       u_key.fib_index = match_local->fib_index;
1528       kv.key = u_key.as_u64;
1529       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1530         {
1531           u = pool_elt_at_index (tsm->users, value.value);
1532           if (u->nstaticsessions)
1533             {
1534               head_index = u->sessions_per_user_list_head_index;
1535               head = pool_elt_at_index (tsm->list_pool, head_index);
1536               elt_index = head->next;
1537               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1538               ses_index = elt->value;
1539               while (ses_index != ~0)
1540                 {
1541                   s = pool_elt_at_index (tsm->sessions, ses_index);
1542                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1543                   ses_index = elt->value;
1544
1545                   if (!(is_lb_session (s)))
1546                     continue;
1547
1548                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1549                       (clib_net_to_host_u16 (s->in2out.port) !=
1550                        match_local->port))
1551                     continue;
1552
1553                   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1554                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1555                 }
1556             }
1557         }
1558
1559       pool_put (m->locals, match_local);
1560     }
1561
1562   vec_free (m->workers);
1563
1564   /* *INDENT-OFF* */
1565   pool_foreach (local, m->locals,
1566   ({
1567     vec_add1 (locals, local - m->locals);
1568     if (sm->num_workers > 1)
1569       {
1570         ip4_header_t ip;
1571         ip.src_address.as_u32 = local->addr.as_u32,
1572         bitmap = clib_bitmap_set (bitmap,
1573                                   sm->worker_in2out_cb (&ip, local->fib_index),
1574                                   1);
1575       }
1576   }));
1577   /* *INDENT-ON* */
1578
1579   ASSERT (vec_len (locals) > 1);
1580
1581   local = pool_elt_at_index (m->locals, locals[0]);
1582   local->prefix = local->probability;
1583   for (i = 1; i < vec_len (locals); i++)
1584     {
1585       local = pool_elt_at_index (m->locals, locals[i]);
1586       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1587       local->prefix = local->probability + prev_local->prefix;
1588     }
1589
1590   /* Assign workers */
1591   if (sm->num_workers > 1)
1592     {
1593       /* *INDENT-OFF* */
1594       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1595       /* *INDENT-ON* */
1596     }
1597
1598   return 0;
1599 }
1600
1601 int
1602 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1603                   u8 twice_nat)
1604 {
1605   snat_address_t *a = 0;
1606   snat_session_t *ses;
1607   u32 *ses_to_be_removed = 0, *ses_index;
1608   snat_main_per_thread_data_t *tsm;
1609   snat_static_mapping_t *m;
1610   snat_interface_t *interface;
1611   int i;
1612   snat_address_t *addresses =
1613     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1614
1615   /* Find SNAT address */
1616   for (i = 0; i < vec_len (addresses); i++)
1617     {
1618       if (addresses[i].addr.as_u32 == addr.as_u32)
1619         {
1620           a = addresses + i;
1621           break;
1622         }
1623     }
1624   if (!a)
1625     return VNET_API_ERROR_NO_SUCH_ENTRY;
1626
1627   if (delete_sm)
1628     {
1629       /* *INDENT-OFF* */
1630       pool_foreach (m, sm->static_mappings,
1631       ({
1632           if (m->external_addr.as_u32 == addr.as_u32)
1633             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1634                                             m->local_port, m->external_port,
1635                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1636                                             m->proto, 0, m->twice_nat,
1637                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1638       }));
1639       /* *INDENT-ON* */
1640     }
1641   else
1642     {
1643       /* Check if address is used in some static mapping */
1644       if (is_snat_address_used_in_static_mapping (sm, addr))
1645         {
1646           nat_log_notice ("address used in static mapping");
1647           return VNET_API_ERROR_UNSPECIFIED;
1648         }
1649     }
1650
1651   if (a->fib_index != ~0)
1652     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1653
1654   /* Delete sessions using address */
1655   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1656     {
1657       /* *INDENT-OFF* */
1658       vec_foreach (tsm, sm->per_thread_data)
1659         {
1660           pool_foreach (ses, tsm->sessions, ({
1661             if (ses->out2in.addr.as_u32 == addr.as_u32)
1662               {
1663                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1664                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1665               }
1666           }));
1667
1668           vec_foreach (ses_index, ses_to_be_removed)
1669             {
1670               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1671               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1672             }
1673
1674           vec_free (ses_to_be_removed);
1675         }
1676       /* *INDENT-ON* */
1677     }
1678
1679 #define _(N, i, n, s) \
1680   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1681   vec_free (a->busy_##n##_ports_per_thread);
1682   foreach_snat_protocol
1683 #undef _
1684     if (twice_nat)
1685     {
1686       vec_del1 (sm->twice_nat_addresses, i);
1687       return 0;
1688     }
1689   else
1690     vec_del1 (sm->addresses, i);
1691
1692   /* Delete external address from FIB */
1693   /* *INDENT-OFF* */
1694   pool_foreach (interface, sm->interfaces,
1695   ({
1696     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1697       continue;
1698
1699     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1700     break;
1701   }));
1702   pool_foreach (interface, sm->output_feature_interfaces,
1703   ({
1704     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1705       continue;
1706
1707     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1708     break;
1709   }));
1710   /* *INDENT-ON* */
1711
1712   return 0;
1713 }
1714
1715 int
1716 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1717 {
1718   snat_main_t *sm = &snat_main;
1719   snat_interface_t *i;
1720   const char *feature_name, *del_feature_name;
1721   snat_address_t *ap;
1722   snat_static_mapping_t *m;
1723   snat_det_map_t *dm;
1724   nat_outside_fib_t *outside_fib;
1725   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1726                                                        sw_if_index);
1727
1728   if (sm->out2in_dpo && !is_inside)
1729     return VNET_API_ERROR_UNSUPPORTED;
1730
1731   /* *INDENT-OFF* */
1732   pool_foreach (i, sm->output_feature_interfaces,
1733   ({
1734     if (i->sw_if_index == sw_if_index)
1735       return VNET_API_ERROR_VALUE_EXIST;
1736   }));
1737   /* *INDENT-ON* */
1738
1739   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1740     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1741   else
1742     {
1743       if (sm->num_workers > 1 && !sm->deterministic)
1744         feature_name =
1745           is_inside ? "nat44-in2out-worker-handoff" :
1746           "nat44-out2in-worker-handoff";
1747       else if (sm->deterministic)
1748         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1749       else if (sm->endpoint_dependent)
1750         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1751       else
1752         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1753     }
1754
1755   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1756     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1757                                                       NAT_FQ_NELTS);
1758
1759   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1760     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1761                                                       NAT_FQ_NELTS);
1762
1763   if (!is_inside)
1764     {
1765       /* *INDENT-OFF* */
1766       vec_foreach (outside_fib, sm->outside_fibs)
1767         {
1768           if (outside_fib->fib_index == fib_index)
1769             {
1770               if (is_del)
1771                 {
1772                   outside_fib->refcount--;
1773                   if (!outside_fib->refcount)
1774                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1775                 }
1776               else
1777                 outside_fib->refcount++;
1778               goto feature_set;
1779             }
1780         }
1781       /* *INDENT-ON* */
1782       if (!is_del)
1783         {
1784           vec_add2 (sm->outside_fibs, outside_fib, 1);
1785           outside_fib->refcount = 1;
1786           outside_fib->fib_index = fib_index;
1787         }
1788     }
1789 feature_set:
1790   /* *INDENT-OFF* */
1791   pool_foreach (i, sm->interfaces,
1792   ({
1793     if (i->sw_if_index == sw_if_index)
1794       {
1795         if (is_del)
1796           {
1797             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1798               {
1799                 if (is_inside)
1800                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1801                 else
1802                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1803
1804                 if (sm->num_workers > 1 && !sm->deterministic)
1805                   {
1806                     del_feature_name = "nat44-handoff-classify";
1807                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1808                                                  "nat44-out2in-worker-handoff";
1809                   }
1810                 else if (sm->deterministic)
1811                   {
1812                     del_feature_name = "nat44-det-classify";
1813                     feature_name = !is_inside ?  "nat44-det-in2out" :
1814                                                  "nat44-det-out2in";
1815                   }
1816                 else if (sm->endpoint_dependent)
1817                   {
1818                     del_feature_name = "nat44-ed-classify";
1819                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1820                                                  "nat44-ed-out2in";
1821                   }
1822                 else
1823                   {
1824                     del_feature_name = "nat44-classify";
1825                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1826                   }
1827
1828                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1829                                              sw_if_index, 0, 0, 0);
1830                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1831                                              sw_if_index, 1, 0, 0);
1832                 if (!is_inside)
1833                   {
1834                     if (sm->endpoint_dependent)
1835                       vnet_feature_enable_disable ("ip4-local",
1836                                                    "nat44-ed-hairpinning",
1837                                                    sw_if_index, 1, 0, 0);
1838                     else if (!sm->deterministic)
1839                       vnet_feature_enable_disable ("ip4-local",
1840                                                    "nat44-hairpinning",
1841                                                    sw_if_index, 1, 0, 0);
1842                   }
1843               }
1844             else
1845               {
1846                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1847                                              sw_if_index, 0, 0, 0);
1848                 pool_put (sm->interfaces, i);
1849                 if (is_inside)
1850                   {
1851                     if (sm->endpoint_dependent)
1852                       vnet_feature_enable_disable ("ip4-local",
1853                                                    "nat44-ed-hairpinning",
1854                                                    sw_if_index, 0, 0, 0);
1855                     else if (!sm->deterministic)
1856                       vnet_feature_enable_disable ("ip4-local",
1857                                                    "nat44-hairpinning",
1858                                                    sw_if_index, 0, 0, 0);
1859                   }
1860               }
1861           }
1862         else
1863           {
1864             if ((nat_interface_is_inside(i) && is_inside) ||
1865                 (nat_interface_is_outside(i) && !is_inside))
1866               return 0;
1867
1868             if (sm->num_workers > 1 && !sm->deterministic)
1869               {
1870                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1871                                                  "nat44-out2in-worker-handoff";
1872                 feature_name = "nat44-handoff-classify";
1873               }
1874             else if (sm->deterministic)
1875               {
1876                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1877                                                  "nat44-det-out2in";
1878                 feature_name = "nat44-det-classify";
1879               }
1880             else if (sm->endpoint_dependent)
1881               {
1882                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1883                                                  "nat44-ed-out2in";
1884                 feature_name = "nat44-ed-classify";
1885               }
1886             else
1887               {
1888                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1889                 feature_name = "nat44-classify";
1890               }
1891
1892             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1893                                          sw_if_index, 0, 0, 0);
1894             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1895                                          sw_if_index, 1, 0, 0);
1896             if (!is_inside)
1897               {
1898                 if (sm->endpoint_dependent)
1899                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1900                                                sw_if_index, 0, 0, 0);
1901                 else if (!sm->deterministic)
1902                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1903                                                sw_if_index, 0, 0, 0);
1904               }
1905             goto set_flags;
1906           }
1907
1908         goto fib;
1909       }
1910   }));
1911   /* *INDENT-ON* */
1912
1913   if (is_del)
1914     return VNET_API_ERROR_NO_SUCH_ENTRY;
1915
1916   pool_get (sm->interfaces, i);
1917   i->sw_if_index = sw_if_index;
1918   i->flags = 0;
1919   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1920                                0);
1921
1922   if (is_inside && !sm->out2in_dpo)
1923     {
1924       if (sm->endpoint_dependent)
1925         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1926                                      sw_if_index, 1, 0, 0);
1927       else if (!sm->deterministic)
1928         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1929                                      sw_if_index, 1, 0, 0);
1930     }
1931
1932 set_flags:
1933   if (is_inside)
1934     {
1935       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1936       return 0;
1937     }
1938   else
1939     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1940
1941   /* Add/delete external addresses to FIB */
1942 fib:
1943   /* *INDENT-OFF* */
1944   vec_foreach (ap, sm->addresses)
1945     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1946
1947   pool_foreach (m, sm->static_mappings,
1948   ({
1949     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1950       continue;
1951
1952     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1953   }));
1954
1955   pool_foreach (dm, sm->det_maps,
1956   ({
1957     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1958   }));
1959   /* *INDENT-ON* */
1960
1961   return 0;
1962 }
1963
1964 int
1965 snat_interface_add_del_output_feature (u32 sw_if_index,
1966                                        u8 is_inside, int is_del)
1967 {
1968   snat_main_t *sm = &snat_main;
1969   snat_interface_t *i;
1970   snat_address_t *ap;
1971   snat_static_mapping_t *m;
1972   nat_outside_fib_t *outside_fib;
1973   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1974                                                        sw_if_index);
1975
1976
1977   if (sm->deterministic ||
1978       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1979     return VNET_API_ERROR_UNSUPPORTED;
1980
1981   /* *INDENT-OFF* */
1982   pool_foreach (i, sm->interfaces,
1983   ({
1984     if (i->sw_if_index == sw_if_index)
1985       return VNET_API_ERROR_VALUE_EXIST;
1986   }));
1987   /* *INDENT-ON* */
1988
1989   if (!is_inside)
1990     {
1991       /* *INDENT-OFF* */
1992       vec_foreach (outside_fib, sm->outside_fibs)
1993         {
1994           if (outside_fib->fib_index == fib_index)
1995             {
1996               if (is_del)
1997                 {
1998                   outside_fib->refcount--;
1999                   if (!outside_fib->refcount)
2000                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2001                 }
2002               else
2003                 outside_fib->refcount++;
2004               goto feature_set;
2005             }
2006         }
2007       /* *INDENT-ON* */
2008       if (!is_del)
2009         {
2010           vec_add2 (sm->outside_fibs, outside_fib, 1);
2011           outside_fib->refcount = 1;
2012           outside_fib->fib_index = fib_index;
2013         }
2014     }
2015
2016 feature_set:
2017   if (is_inside)
2018     {
2019       if (sm->endpoint_dependent)
2020         {
2021           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2022                                        sw_if_index, !is_del, 0, 0);
2023           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2024                                        sw_if_index, !is_del, 0, 0);
2025         }
2026       else
2027         {
2028           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2029                                        sw_if_index, !is_del, 0, 0);
2030           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2031                                        sw_if_index, !is_del, 0, 0);
2032         }
2033       goto fq;
2034     }
2035
2036   if (sm->num_workers > 1)
2037     {
2038       vnet_feature_enable_disable ("ip4-unicast",
2039                                    "nat44-out2in-worker-handoff",
2040                                    sw_if_index, !is_del, 0, 0);
2041       vnet_feature_enable_disable ("ip4-output",
2042                                    "nat44-in2out-output-worker-handoff",
2043                                    sw_if_index, !is_del, 0, 0);
2044     }
2045   else
2046     {
2047       if (sm->endpoint_dependent)
2048         {
2049           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2050                                        sw_if_index, !is_del, 0, 0);
2051           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2052                                        sw_if_index, !is_del, 0, 0);
2053         }
2054       else
2055         {
2056           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2057                                        sw_if_index, !is_del, 0, 0);
2058           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2059                                        sw_if_index, !is_del, 0, 0);
2060         }
2061     }
2062
2063 fq:
2064   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2065     sm->fq_in2out_output_index =
2066       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2067
2068   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2069     sm->fq_out2in_index =
2070       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2071
2072   /* *INDENT-OFF* */
2073   pool_foreach (i, sm->output_feature_interfaces,
2074   ({
2075     if (i->sw_if_index == sw_if_index)
2076       {
2077         if (is_del)
2078           pool_put (sm->output_feature_interfaces, i);
2079         else
2080           return VNET_API_ERROR_VALUE_EXIST;
2081
2082         goto fib;
2083       }
2084   }));
2085   /* *INDENT-ON* */
2086
2087   if (is_del)
2088     return VNET_API_ERROR_NO_SUCH_ENTRY;
2089
2090   pool_get (sm->output_feature_interfaces, i);
2091   i->sw_if_index = sw_if_index;
2092   i->flags = 0;
2093   if (is_inside)
2094     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2095   else
2096     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2097
2098   /* Add/delete external addresses to FIB */
2099 fib:
2100   if (is_inside)
2101     return 0;
2102
2103   /* *INDENT-OFF* */
2104   vec_foreach (ap, sm->addresses)
2105     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2106
2107   pool_foreach (m, sm->static_mappings,
2108   ({
2109     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2110       continue;
2111
2112     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2113   }));
2114   /* *INDENT-ON* */
2115
2116   return 0;
2117 }
2118
2119 int
2120 snat_set_workers (uword * bitmap)
2121 {
2122   snat_main_t *sm = &snat_main;
2123   int i, j = 0;
2124
2125   if (sm->num_workers < 2)
2126     return VNET_API_ERROR_FEATURE_DISABLED;
2127
2128   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2129     return VNET_API_ERROR_INVALID_WORKER;
2130
2131   vec_free (sm->workers);
2132   /* *INDENT-OFF* */
2133   clib_bitmap_foreach (i, bitmap,
2134     ({
2135       vec_add1(sm->workers, i);
2136       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2137       j++;
2138     }));
2139   /* *INDENT-ON* */
2140
2141   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2142   sm->num_snat_thread = _vec_len (sm->workers);
2143
2144   return 0;
2145 }
2146
2147 static void
2148 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2149                          u32 old_fib_index)
2150 {
2151   snat_main_t *sm = &snat_main;
2152   nat_outside_fib_t *outside_fib;
2153   snat_interface_t *i;
2154   u8 is_add = 1;
2155
2156   if (new_fib_index == old_fib_index)
2157     return;
2158
2159   if (!vec_len (sm->outside_fibs))
2160     return;
2161
2162   pool_foreach (i, sm->interfaces, (
2163                                      {
2164                                      if (i->sw_if_index == sw_if_index)
2165                                      {
2166                                      if (!(nat_interface_is_outside (i)))
2167                                      return;}
2168                                      }
2169                 ));
2170   vec_foreach (outside_fib, sm->outside_fibs)
2171   {
2172     if (outside_fib->fib_index == old_fib_index)
2173       {
2174         outside_fib->refcount--;
2175         if (!outside_fib->refcount)
2176           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2177         break;
2178       }
2179   }
2180
2181   vec_foreach (outside_fib, sm->outside_fibs)
2182   {
2183     if (outside_fib->fib_index == new_fib_index)
2184       {
2185         outside_fib->refcount++;
2186         is_add = 0;
2187         break;
2188       }
2189   }
2190
2191   if (is_add)
2192     {
2193       vec_add2 (sm->outside_fibs, outside_fib, 1);
2194       outside_fib->refcount = 1;
2195       outside_fib->fib_index = new_fib_index;
2196     }
2197 }
2198
2199 static void
2200 snat_ip4_table_bind (ip4_main_t * im,
2201                      uword opaque,
2202                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2203 {
2204   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2205 }
2206
2207 static void
2208 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2209                                        uword opaque,
2210                                        u32 sw_if_index,
2211                                        ip4_address_t * address,
2212                                        u32 address_length,
2213                                        u32 if_address_index, u32 is_delete);
2214
2215 static void
2216 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2217                                  uword opaque,
2218                                  u32 sw_if_index,
2219                                  ip4_address_t * address,
2220                                  u32 address_length,
2221                                  u32 if_address_index, u32 is_delete);
2222
2223 static int
2224 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2225                                  u32 fib_index,
2226                                  u32 thread_index,
2227                                  snat_session_key_t * k,
2228                                  u16 port_per_thread, u32 snat_thread_index);
2229
2230 static clib_error_t *
2231 snat_init (vlib_main_t * vm)
2232 {
2233   snat_main_t *sm = &snat_main;
2234   clib_error_t *error = 0;
2235   ip4_main_t *im = &ip4_main;
2236   ip_lookup_main_t *lm = &im->lookup_main;
2237   uword *p;
2238   vlib_thread_registration_t *tr;
2239   vlib_thread_main_t *tm = vlib_get_thread_main ();
2240   uword *bitmap = 0;
2241   u32 i;
2242   ip4_add_del_interface_address_callback_t cb4;
2243   vlib_node_t *error_drop_node;
2244
2245   sm->vlib_main = vm;
2246   sm->vnet_main = vnet_get_main ();
2247   sm->ip4_main = im;
2248   sm->ip4_lookup_main = lm;
2249   sm->api_main = &api_main;
2250   sm->first_worker_index = 0;
2251   sm->num_workers = 0;
2252   sm->num_snat_thread = 1;
2253   sm->workers = 0;
2254   sm->port_per_thread = 0xffff - 1024;
2255   sm->fq_in2out_index = ~0;
2256   sm->fq_out2in_index = ~0;
2257   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2258   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2259   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2260   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2261   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2262   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2263   sm->forwarding_enabled = 0;
2264   sm->log_class = vlib_log_register_class ("nat", 0);
2265   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2266   sm->error_node_index = error_drop_node->index;
2267   sm->mss_clamping = 0;
2268
2269   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2270   if (p)
2271     {
2272       tr = (vlib_thread_registration_t *) p[0];
2273       if (tr)
2274         {
2275           sm->num_workers = tr->count;
2276           sm->first_worker_index = tr->first_index;
2277         }
2278     }
2279
2280   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2281
2282   /* Use all available workers by default */
2283   if (sm->num_workers > 1)
2284     {
2285       for (i = 0; i < sm->num_workers; i++)
2286         bitmap = clib_bitmap_set (bitmap, i, 1);
2287       snat_set_workers (bitmap);
2288       clib_bitmap_free (bitmap);
2289     }
2290   else
2291     {
2292       sm->per_thread_data[0].snat_thread_index = 0;
2293     }
2294
2295   error = snat_api_init (vm, sm);
2296   if (error)
2297     return error;
2298
2299   /* Set up the interface address add/del callback */
2300   cb4.function = snat_ip4_add_del_interface_address_cb;
2301   cb4.function_opaque = 0;
2302
2303   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2304
2305   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2306   cb4.function_opaque = 0;
2307
2308   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2309
2310   nat_dpo_module_init ();
2311
2312   /* Init counters */
2313   sm->total_users.name = "total-users";
2314   sm->total_users.stat_segment_name = "/nat44/total-users";
2315   vlib_validate_simple_counter (&sm->total_users, 0);
2316   vlib_zero_simple_counter (&sm->total_users, 0);
2317   sm->total_sessions.name = "total-sessions";
2318   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2319   vlib_validate_simple_counter (&sm->total_sessions, 0);
2320   vlib_zero_simple_counter (&sm->total_sessions, 0);
2321
2322   /* Init IPFIX logging */
2323   snat_ipfix_logging_init (vm);
2324
2325   /* Init NAT64 */
2326   error = nat64_init (vm);
2327   if (error)
2328     return error;
2329
2330   dslite_init (vm);
2331
2332   nat66_init ();
2333
2334   ip4_table_bind_callback_t cbt4 = {
2335     .function = snat_ip4_table_bind,
2336   };
2337   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2338
2339   /* Init virtual fragmenentation reassembly */
2340   return nat_reass_init (vm);
2341 }
2342
2343 VLIB_INIT_FUNCTION (snat_init);
2344
2345 void
2346 snat_free_outside_address_and_port (snat_address_t * addresses,
2347                                     u32 thread_index, snat_session_key_t * k)
2348 {
2349   snat_address_t *a;
2350   u32 address_index;
2351   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2352
2353   for (address_index = 0; address_index < vec_len (addresses);
2354        address_index++)
2355     {
2356       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2357         break;
2358     }
2359
2360   ASSERT (address_index < vec_len (addresses));
2361
2362   a = addresses + address_index;
2363
2364   switch (k->protocol)
2365     {
2366 #define _(N, i, n, s) \
2367     case SNAT_PROTOCOL_##N: \
2368       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2369         port_host_byte_order) == 1); \
2370       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2371         port_host_byte_order, 0); \
2372       a->busy_##n##_ports--; \
2373       a->busy_##n##_ports_per_thread[thread_index]--; \
2374       break;
2375       foreach_snat_protocol
2376 #undef _
2377     default:
2378       nat_log_info ("unknown protocol");
2379       return;
2380     }
2381 }
2382
2383 int
2384 snat_static_mapping_match (snat_main_t * sm,
2385                            snat_session_key_t match,
2386                            snat_session_key_t * mapping,
2387                            u8 by_external,
2388                            u8 * is_addr_only,
2389                            twice_nat_type_t * twice_nat,
2390                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2391                            u8 * is_identity_nat)
2392 {
2393   clib_bihash_kv_8_8_t kv, value;
2394   snat_static_mapping_t *m;
2395   snat_session_key_t m_key;
2396   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2397   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2398   u8 backend_index;
2399   nat44_lb_addr_port_t *local;
2400
2401   m_key.fib_index = match.fib_index;
2402   if (by_external)
2403     {
2404       mapping_hash = &sm->static_mapping_by_external;
2405       m_key.fib_index = 0;
2406     }
2407
2408   m_key.addr = match.addr;
2409   m_key.port = clib_net_to_host_u16 (match.port);
2410   m_key.protocol = match.protocol;
2411
2412   kv.key = m_key.as_u64;
2413
2414   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2415     {
2416       /* Try address only mapping */
2417       m_key.port = 0;
2418       m_key.protocol = 0;
2419       kv.key = m_key.as_u64;
2420       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2421         return 1;
2422     }
2423
2424   m = pool_elt_at_index (sm->static_mappings, value.value);
2425
2426   if (by_external)
2427     {
2428       if (is_lb_static_mapping (m))
2429         {
2430           if (PREDICT_FALSE (lb != 0))
2431             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2432           if (m->affinity)
2433             {
2434               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2435                                               match.protocol, match.port,
2436                                               &backend_index))
2437                 goto get_local;
2438
2439               local = pool_elt_at_index (m->locals, backend_index);
2440               mapping->addr = local->addr;
2441               mapping->port = clib_host_to_net_u16 (local->port);
2442               mapping->fib_index = local->fib_index;
2443               goto end;
2444             }
2445         get_local:
2446           /* *INDENT-OFF* */
2447           pool_foreach_index (i, m->locals,
2448           ({
2449             vec_add1 (tmp, i);
2450           }));
2451           /* *INDENT-ON* */
2452           hi = vec_len (tmp) - 1;
2453           local = pool_elt_at_index (m->locals, tmp[hi]);
2454           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2455           while (lo < hi)
2456             {
2457               mid = ((hi - lo) >> 1) + lo;
2458               local = pool_elt_at_index (m->locals, tmp[mid]);
2459               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2460             }
2461           local = pool_elt_at_index (m->locals, tmp[lo]);
2462           if (!(local->prefix >= rand))
2463             return 1;
2464           if (PREDICT_FALSE (sm->num_workers > 1))
2465             {
2466               ip4_header_t ip = {
2467                 .src_address = local->addr,
2468               };
2469               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2470                   vlib_get_thread_index ())
2471                 goto get_local;
2472             }
2473           mapping->addr = local->addr;
2474           mapping->port = clib_host_to_net_u16 (local->port);
2475           mapping->fib_index = local->fib_index;
2476           if (m->affinity)
2477             {
2478               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2479                                                 match.protocol, match.port,
2480                                                 tmp[lo], m->affinity,
2481                                                 m->affinity_per_service_list_head_index))
2482                 nat_log_info ("create affinity record failed");
2483             }
2484           vec_free (tmp);
2485         }
2486       else
2487         {
2488           if (PREDICT_FALSE (lb != 0))
2489             *lb = NO_LB_NAT;
2490           mapping->fib_index = m->fib_index;
2491           mapping->addr = m->local_addr;
2492           /* Address only mapping doesn't change port */
2493           mapping->port = is_addr_only_static_mapping (m) ? match.port
2494             : clib_host_to_net_u16 (m->local_port);
2495         }
2496       mapping->protocol = m->proto;
2497     }
2498   else
2499     {
2500       mapping->addr = m->external_addr;
2501       /* Address only mapping doesn't change port */
2502       mapping->port = is_addr_only_static_mapping (m) ? match.port
2503         : clib_host_to_net_u16 (m->external_port);
2504       mapping->fib_index = sm->outside_fib_index;
2505     }
2506
2507 end:
2508   if (PREDICT_FALSE (is_addr_only != 0))
2509     *is_addr_only = is_addr_only_static_mapping (m);
2510
2511   if (PREDICT_FALSE (twice_nat != 0))
2512     *twice_nat = m->twice_nat;
2513
2514   if (PREDICT_FALSE (is_identity_nat != 0))
2515     *is_identity_nat = is_identity_static_mapping (m);
2516
2517   return 0;
2518 }
2519
2520 static_always_inline u16
2521 snat_random_port (u16 min, u16 max)
2522 {
2523   snat_main_t *sm = &snat_main;
2524   return min + random_u32 (&sm->random_seed) /
2525     (random_u32_max () / (max - min + 1) + 1);
2526 }
2527
2528 int
2529 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2530                                      u32 fib_index,
2531                                      u32 thread_index,
2532                                      snat_session_key_t * k,
2533                                      u16 port_per_thread,
2534                                      u32 snat_thread_index)
2535 {
2536   snat_main_t *sm = &snat_main;
2537
2538   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2539                                   port_per_thread, snat_thread_index);
2540 }
2541
2542 static int
2543 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2544                                  u32 fib_index,
2545                                  u32 thread_index,
2546                                  snat_session_key_t * k,
2547                                  u16 port_per_thread, u32 snat_thread_index)
2548 {
2549   int i;
2550   snat_address_t *a, *ga = 0;
2551   u32 portnum;
2552
2553   for (i = 0; i < vec_len (addresses); i++)
2554     {
2555       a = addresses + i;
2556       switch (k->protocol)
2557         {
2558 #define _(N, j, n, s) \
2559         case SNAT_PROTOCOL_##N: \
2560           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2561             { \
2562               if (a->fib_index == fib_index) \
2563                 { \
2564                   while (1) \
2565                     { \
2566                       portnum = (port_per_thread * \
2567                         snat_thread_index) + \
2568                         snat_random_port(1, port_per_thread) + 1024; \
2569                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2570                         continue; \
2571                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2572                       a->busy_##n##_ports_per_thread[thread_index]++; \
2573                       a->busy_##n##_ports++; \
2574                       k->addr = a->addr; \
2575                       k->port = clib_host_to_net_u16(portnum); \
2576                       return 0; \
2577                     } \
2578                 } \
2579               else if (a->fib_index == ~0) \
2580                 { \
2581                   ga = a; \
2582                 } \
2583             } \
2584           break;
2585           foreach_snat_protocol
2586 #undef _
2587         default:
2588           nat_log_info ("unknown protocol");
2589           return 1;
2590         }
2591
2592     }
2593
2594   if (ga)
2595     {
2596       a = ga;
2597       switch (k->protocol)
2598         {
2599 #define _(N, j, n, s) \
2600         case SNAT_PROTOCOL_##N: \
2601           while (1) \
2602             { \
2603               portnum = (port_per_thread * \
2604                 snat_thread_index) + \
2605                 snat_random_port(1, port_per_thread) + 1024; \
2606               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2607                 continue; \
2608               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2609               a->busy_##n##_ports_per_thread[thread_index]++; \
2610               a->busy_##n##_ports++; \
2611               k->addr = a->addr; \
2612               k->port = clib_host_to_net_u16(portnum); \
2613               return 0; \
2614             }
2615           break;
2616           foreach_snat_protocol
2617 #undef _
2618         default:
2619           nat_log_info ("unknown protocol");
2620           return 1;
2621         }
2622     }
2623
2624   /* Totally out of translations to use... */
2625   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2626   return 1;
2627 }
2628
2629 static int
2630 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2631                               u32 fib_index,
2632                               u32 thread_index,
2633                               snat_session_key_t * k,
2634                               u16 port_per_thread, u32 snat_thread_index)
2635 {
2636   snat_main_t *sm = &snat_main;
2637   snat_address_t *a = addresses;
2638   u16 m, ports, portnum, A, j;
2639   m = 16 - (sm->psid_offset + sm->psid_length);
2640   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2641
2642   if (!vec_len (addresses))
2643     goto exhausted;
2644
2645   switch (k->protocol)
2646     {
2647 #define _(N, i, n, s) \
2648     case SNAT_PROTOCOL_##N: \
2649       if (a->busy_##n##_ports < ports) \
2650         { \
2651           while (1) \
2652             { \
2653               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2654               j = snat_random_port(0, pow2_mask(m)); \
2655               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2656               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2657                 continue; \
2658               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2659               a->busy_##n##_ports++; \
2660               k->addr = a->addr; \
2661               k->port = clib_host_to_net_u16 (portnum); \
2662               return 0; \
2663             } \
2664         } \
2665       break;
2666       foreach_snat_protocol
2667 #undef _
2668     default:
2669       nat_log_info ("unknown protocol");
2670       return 1;
2671     }
2672
2673 exhausted:
2674   /* Totally out of translations to use... */
2675   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2676   return 1;
2677 }
2678
2679 static int
2680 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2681                                u32 fib_index,
2682                                u32 thread_index,
2683                                snat_session_key_t * k,
2684                                u16 port_per_thread, u32 snat_thread_index)
2685 {
2686   snat_main_t *sm = &snat_main;
2687   snat_address_t *a = addresses;
2688   u16 portnum, ports;
2689
2690   ports = sm->end_port - sm->start_port + 1;
2691
2692   if (!vec_len (addresses))
2693     goto exhausted;
2694
2695   switch (k->protocol)
2696     {
2697 #define _(N, i, n, s) \
2698     case SNAT_PROTOCOL_##N: \
2699       if (a->busy_##n##_ports < ports) \
2700         { \
2701           while (1) \
2702             { \
2703               portnum = snat_random_port(sm->start_port, sm->end_port); \
2704               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2705                 continue; \
2706               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2707               a->busy_##n##_ports++; \
2708               k->addr = a->addr; \
2709               k->port = clib_host_to_net_u16 (portnum); \
2710               return 0; \
2711             } \
2712         } \
2713       break;
2714       foreach_snat_protocol
2715 #undef _
2716     default:
2717       nat_log_info ("unknown protocol");
2718       return 1;
2719     }
2720
2721 exhausted:
2722   /* Totally out of translations to use... */
2723   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2724   return 1;
2725 }
2726
2727 void
2728 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2729 {
2730   dpo_id_t dpo_v4 = DPO_INVALID;
2731   fib_prefix_t pfx = {
2732     .fp_proto = FIB_PROTOCOL_IP4,
2733     .fp_len = 32,
2734     .fp_addr.ip4.as_u32 = addr.as_u32,
2735   };
2736
2737   if (is_add)
2738     {
2739       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2740       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2741                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2742       dpo_reset (&dpo_v4);
2743     }
2744   else
2745     {
2746       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2747     }
2748 }
2749
2750 u8 *
2751 format_session_kvp (u8 * s, va_list * args)
2752 {
2753   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2754   snat_session_key_t k;
2755
2756   k.as_u64 = v->key;
2757
2758   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2759
2760   return s;
2761 }
2762
2763 u8 *
2764 format_static_mapping_kvp (u8 * s, va_list * args)
2765 {
2766   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2767   snat_session_key_t k;
2768
2769   k.as_u64 = v->key;
2770
2771   s = format (s, "%U static-mapping-index %llu",
2772               format_static_mapping_key, &k, v->value);
2773
2774   return s;
2775 }
2776
2777 u8 *
2778 format_user_kvp (u8 * s, va_list * args)
2779 {
2780   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2781   snat_user_key_t k;
2782
2783   k.as_u64 = v->key;
2784
2785   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2786               k.fib_index, v->value);
2787
2788   return s;
2789 }
2790
2791 u8 *
2792 format_ed_session_kvp (u8 * s, va_list * args)
2793 {
2794   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2795   nat_ed_ses_key_t k;
2796
2797   k.as_u64[0] = v->key[0];
2798   k.as_u64[1] = v->key[1];
2799
2800   s =
2801     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2802             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2803             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2804             format_ip_protocol, k.proto, k.fib_index, v->value);
2805
2806   return s;
2807 }
2808
2809 static u32
2810 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2811 {
2812   snat_main_t *sm = &snat_main;
2813   u32 next_worker_index = 0;
2814   u32 hash;
2815
2816   next_worker_index = sm->first_worker_index;
2817   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2818     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2819
2820   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2821     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2822   else
2823     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2824
2825   return next_worker_index;
2826 }
2827
2828 static u32
2829 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2830 {
2831   snat_main_t *sm = &snat_main;
2832   udp_header_t *udp;
2833   u16 port;
2834   snat_session_key_t m_key;
2835   clib_bihash_kv_8_8_t kv, value;
2836   snat_static_mapping_t *m;
2837   u32 proto;
2838   u32 next_worker_index = 0;
2839
2840   /* first try static mappings without port */
2841   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2842     {
2843       m_key.addr = ip0->dst_address;
2844       m_key.port = 0;
2845       m_key.protocol = 0;
2846       m_key.fib_index = rx_fib_index0;
2847       kv.key = m_key.as_u64;
2848       if (!clib_bihash_search_8_8
2849           (&sm->static_mapping_by_external, &kv, &value))
2850         {
2851           m = pool_elt_at_index (sm->static_mappings, value.value);
2852           return m->workers[0];
2853         }
2854     }
2855
2856   proto = ip_proto_to_snat_proto (ip0->protocol);
2857   udp = ip4_next_header (ip0);
2858   port = udp->dst_port;
2859
2860   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2861     {
2862       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2863         return vlib_get_thread_index ();
2864
2865       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2866         {
2867           nat_reass_ip4_t *reass;
2868
2869           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2870                                       ip0->fragment_id, ip0->protocol);
2871
2872           if (reass && (reass->thread_index != (u32) ~ 0))
2873             return reass->thread_index;
2874           else
2875             return vlib_get_thread_index ();
2876         }
2877     }
2878
2879   /* unknown protocol */
2880   if (PREDICT_FALSE (proto == ~0))
2881     {
2882       /* use current thread */
2883       return vlib_get_thread_index ();
2884     }
2885
2886   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2887     {
2888       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2889       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2890       if (!icmp_is_error_message (icmp))
2891         port = echo->identifier;
2892       else
2893         {
2894           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2895           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2896           void *l4_header = ip4_next_header (inner_ip);
2897           switch (proto)
2898             {
2899             case SNAT_PROTOCOL_ICMP:
2900               icmp = (icmp46_header_t *) l4_header;
2901               echo = (icmp_echo_header_t *) (icmp + 1);
2902               port = echo->identifier;
2903               break;
2904             case SNAT_PROTOCOL_UDP:
2905             case SNAT_PROTOCOL_TCP:
2906               port = ((tcp_udp_header_t *) l4_header)->src_port;
2907               break;
2908             default:
2909               return vlib_get_thread_index ();
2910             }
2911         }
2912     }
2913
2914   /* try static mappings with port */
2915   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2916     {
2917       m_key.addr = ip0->dst_address;
2918       m_key.port = clib_net_to_host_u16 (port);
2919       m_key.protocol = proto;
2920       m_key.fib_index = rx_fib_index0;
2921       kv.key = m_key.as_u64;
2922       if (!clib_bihash_search_8_8
2923           (&sm->static_mapping_by_external, &kv, &value))
2924         {
2925           m = pool_elt_at_index (sm->static_mappings, value.value);
2926           return m->workers[0];
2927         }
2928     }
2929
2930   /* worker by outside port */
2931   next_worker_index = sm->first_worker_index;
2932   next_worker_index +=
2933     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2934   return next_worker_index;
2935 }
2936
2937 static u32
2938 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2939 {
2940   snat_main_t *sm = &snat_main;
2941   clib_bihash_kv_8_8_t kv, value;
2942   u32 proto, next_worker_index = 0;
2943   udp_header_t *udp;
2944   u16 port;
2945   snat_static_mapping_t *m;
2946   u32 hash;
2947
2948   /* first try static mappings without port */
2949   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2950     {
2951       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2952       if (!clib_bihash_search_8_8
2953           (&sm->static_mapping_by_external, &kv, &value))
2954         {
2955           m = pool_elt_at_index (sm->static_mappings, value.value);
2956           return m->workers[0];
2957         }
2958     }
2959
2960   proto = ip_proto_to_snat_proto (ip->protocol);
2961
2962   /* unknown protocol */
2963   if (PREDICT_FALSE (proto == ~0))
2964     {
2965       /* use current thread */
2966       return vlib_get_thread_index ();
2967     }
2968
2969   udp = ip4_next_header (ip);
2970   port = udp->dst_port;
2971
2972   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2973     {
2974       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2975       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2976       if (!icmp_is_error_message (icmp))
2977         port = echo->identifier;
2978       else
2979         {
2980           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2981           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2982           void *l4_header = ip4_next_header (inner_ip);
2983           switch (proto)
2984             {
2985             case SNAT_PROTOCOL_ICMP:
2986               icmp = (icmp46_header_t *) l4_header;
2987               echo = (icmp_echo_header_t *) (icmp + 1);
2988               port = echo->identifier;
2989               break;
2990             case SNAT_PROTOCOL_UDP:
2991             case SNAT_PROTOCOL_TCP:
2992               port = ((tcp_udp_header_t *) l4_header)->src_port;
2993               break;
2994             default:
2995               return vlib_get_thread_index ();
2996             }
2997         }
2998     }
2999
3000   /* try static mappings with port */
3001   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3002     {
3003       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
3004                   clib_net_to_host_u16 (port));
3005       if (!clib_bihash_search_8_8
3006           (&sm->static_mapping_by_external, &kv, &value))
3007         {
3008           m = pool_elt_at_index (sm->static_mappings, value.value);
3009           if (!is_lb_static_mapping (m))
3010             return m->workers[0];
3011
3012           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3013             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3014
3015           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3016             return m->workers[hash & (_vec_len (m->workers) - 1)];
3017           else
3018             return m->workers[hash % _vec_len (m->workers)];
3019         }
3020     }
3021
3022   /* worker by outside port */
3023   next_worker_index = sm->first_worker_index;
3024   next_worker_index +=
3025     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3026
3027   return next_worker_index;
3028 }
3029
3030 static clib_error_t *
3031 snat_config (vlib_main_t * vm, unformat_input_t * input)
3032 {
3033   snat_main_t *sm = &snat_main;
3034   nat66_main_t *nm = &nat66_main;
3035   u32 translation_buckets = 1024;
3036   u32 translation_memory_size = 128 << 20;
3037   u32 user_buckets = 128;
3038   u32 user_memory_size = 64 << 20;
3039   u32 max_translations_per_user = 100;
3040   u32 outside_vrf_id = 0;
3041   u32 outside_ip6_vrf_id = 0;
3042   u32 inside_vrf_id = 0;
3043   u32 static_mapping_buckets = 1024;
3044   u32 static_mapping_memory_size = 64 << 20;
3045   u32 nat64_bib_buckets = 1024;
3046   u32 nat64_bib_memory_size = 128 << 20;
3047   u32 nat64_st_buckets = 2048;
3048   u32 nat64_st_memory_size = 256 << 20;
3049   u8 static_mapping_only = 0;
3050   u8 static_mapping_connection_tracking = 0;
3051   snat_main_per_thread_data_t *tsm;
3052   dslite_main_t *dm = &dslite_main;
3053
3054   sm->deterministic = 0;
3055   sm->out2in_dpo = 0;
3056   sm->endpoint_dependent = 0;
3057
3058   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3059     {
3060       if (unformat
3061           (input, "translation hash buckets %d", &translation_buckets))
3062         ;
3063       else if (unformat (input, "translation hash memory %d",
3064                          &translation_memory_size));
3065       else if (unformat (input, "user hash buckets %d", &user_buckets))
3066         ;
3067       else if (unformat (input, "user hash memory %d", &user_memory_size))
3068         ;
3069       else if (unformat (input, "max translations per user %d",
3070                          &max_translations_per_user))
3071         ;
3072       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3073         ;
3074       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3075         ;
3076       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3077         ;
3078       else if (unformat (input, "static mapping only"))
3079         {
3080           static_mapping_only = 1;
3081           if (unformat (input, "connection tracking"))
3082             static_mapping_connection_tracking = 1;
3083         }
3084       else if (unformat (input, "deterministic"))
3085         sm->deterministic = 1;
3086       else if (unformat (input, "nat64 bib hash buckets %d",
3087                          &nat64_bib_buckets))
3088         ;
3089       else if (unformat (input, "nat64 bib hash memory %d",
3090                          &nat64_bib_memory_size))
3091         ;
3092       else
3093         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3094         ;
3095       else if (unformat (input, "nat64 st hash memory %d",
3096                          &nat64_st_memory_size))
3097         ;
3098       else if (unformat (input, "out2in dpo"))
3099         sm->out2in_dpo = 1;
3100       else if (unformat (input, "dslite ce"))
3101         dslite_set_ce (dm, 1);
3102       else if (unformat (input, "endpoint-dependent"))
3103         sm->endpoint_dependent = 1;
3104       else
3105         return clib_error_return (0, "unknown input '%U'",
3106                                   format_unformat_error, input);
3107     }
3108
3109   if (sm->deterministic && sm->endpoint_dependent)
3110     return clib_error_return (0,
3111                               "deterministic and endpoint-dependent modes are mutually exclusive");
3112
3113   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3114     return clib_error_return (0,
3115                               "static mapping only mode available only for simple nat");
3116
3117   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3118     return clib_error_return (0,
3119                               "out2in dpo mode available only for simple nat");
3120
3121   /* for show commands, etc. */
3122   sm->translation_buckets = translation_buckets;
3123   sm->translation_memory_size = translation_memory_size;
3124   /* do not exceed load factor 10 */
3125   sm->max_translations = 10 * translation_buckets;
3126   sm->user_buckets = user_buckets;
3127   sm->user_memory_size = user_memory_size;
3128   sm->max_translations_per_user = max_translations_per_user;
3129   sm->outside_vrf_id = outside_vrf_id;
3130   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3131                                                              outside_vrf_id,
3132                                                              FIB_SOURCE_PLUGIN_HI);
3133   nm->outside_vrf_id = outside_ip6_vrf_id;
3134   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3135                                                              outside_ip6_vrf_id,
3136                                                              FIB_SOURCE_PLUGIN_HI);
3137   sm->inside_vrf_id = inside_vrf_id;
3138   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3139                                                             inside_vrf_id,
3140                                                             FIB_SOURCE_PLUGIN_HI);
3141   sm->static_mapping_only = static_mapping_only;
3142   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3143
3144   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3145                   nat64_st_memory_size);
3146
3147   if (sm->deterministic)
3148     {
3149       sm->in2out_node_index = snat_det_in2out_node.index;
3150       sm->in2out_output_node_index = ~0;
3151       sm->out2in_node_index = snat_det_out2in_node.index;
3152       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3153       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3154     }
3155   else
3156     {
3157       if (sm->endpoint_dependent)
3158         {
3159           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3160           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3161           sm->in2out_node_index = nat44_ed_in2out_node.index;
3162           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3163           sm->out2in_node_index = nat44_ed_out2in_node.index;
3164           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3165           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3166           nat_affinity_init (vm);
3167         }
3168       else
3169         {
3170           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3171           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3172           sm->in2out_node_index = snat_in2out_node.index;
3173           sm->in2out_output_node_index = snat_in2out_output_node.index;
3174           sm->out2in_node_index = snat_out2in_node.index;
3175           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3176           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3177         }
3178       if (!static_mapping_only ||
3179           (static_mapping_only && static_mapping_connection_tracking))
3180         {
3181           /* *INDENT-OFF* */
3182           vec_foreach (tsm, sm->per_thread_data)
3183             {
3184               if (sm->endpoint_dependent)
3185                 {
3186                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3187                                          translation_buckets,
3188                                          translation_memory_size);
3189                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3190                                                       format_ed_session_kvp);
3191
3192                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3193                                          translation_buckets,
3194                                          translation_memory_size);
3195                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3196                                                       format_ed_session_kvp);
3197                 }
3198               else
3199                 {
3200                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3201                                         translation_buckets,
3202                                         translation_memory_size);
3203                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3204                                                      format_session_kvp);
3205
3206                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3207                                         translation_buckets,
3208                                         translation_memory_size);
3209                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3210                                                      format_session_kvp);
3211                 }
3212
3213               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3214                                     user_memory_size);
3215               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3216                                                  format_user_kvp);
3217             }
3218           /* *INDENT-ON* */
3219
3220         }
3221       else
3222         {
3223           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3224           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3225         }
3226       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3227                             "static_mapping_by_local", static_mapping_buckets,
3228                             static_mapping_memory_size);
3229       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3230                                          format_static_mapping_kvp);
3231
3232       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3233                             "static_mapping_by_external",
3234                             static_mapping_buckets,
3235                             static_mapping_memory_size);
3236       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3237                                          format_static_mapping_kvp);
3238     }
3239
3240   return 0;
3241 }
3242
3243 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3244
3245 static void
3246 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3247                                  uword opaque,
3248                                  u32 sw_if_index,
3249                                  ip4_address_t * address,
3250                                  u32 address_length,
3251                                  u32 if_address_index, u32 is_delete)
3252 {
3253   snat_main_t *sm = &snat_main;
3254   snat_static_map_resolve_t *rp;
3255   snat_static_mapping_t *m;
3256   snat_session_key_t m_key;
3257   clib_bihash_kv_8_8_t kv, value;
3258   int i, rv;
3259   ip4_address_t l_addr;
3260
3261   for (i = 0; i < vec_len (sm->to_resolve); i++)
3262     {
3263       rp = sm->to_resolve + i;
3264       if (rp->addr_only == 0)
3265         continue;
3266       if (rp->sw_if_index == sw_if_index)
3267         goto match;
3268     }
3269
3270   return;
3271
3272 match:
3273   m_key.addr.as_u32 = address->as_u32;
3274   m_key.port = rp->addr_only ? 0 : rp->e_port;
3275   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3276   m_key.fib_index = sm->outside_fib_index;
3277   kv.key = m_key.as_u64;
3278   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3279     m = 0;
3280   else
3281     m = pool_elt_at_index (sm->static_mappings, value.value);
3282
3283   if (!is_delete)
3284     {
3285       /* Don't trip over lease renewal, static config */
3286       if (m)
3287         return;
3288     }
3289   else
3290     {
3291       if (!m)
3292         return;
3293     }
3294
3295   /* Indetity mapping? */
3296   if (rp->l_addr.as_u32 == 0)
3297     l_addr.as_u32 = address[0].as_u32;
3298   else
3299     l_addr.as_u32 = rp->l_addr.as_u32;
3300   /* Add the static mapping */
3301   rv = snat_add_static_mapping (l_addr,
3302                                 address[0],
3303                                 rp->l_port,
3304                                 rp->e_port,
3305                                 rp->vrf_id,
3306                                 rp->addr_only, ~0 /* sw_if_index */ ,
3307                                 rp->proto, !is_delete, rp->twice_nat,
3308                                 rp->out2in_only, rp->tag, rp->identity_nat);
3309   if (rv)
3310     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3311 }
3312
3313 static void
3314 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3315                                        uword opaque,
3316                                        u32 sw_if_index,
3317                                        ip4_address_t * address,
3318                                        u32 address_length,
3319                                        u32 if_address_index, u32 is_delete)
3320 {
3321   snat_main_t *sm = &snat_main;
3322   snat_static_map_resolve_t *rp;
3323   ip4_address_t l_addr;
3324   int i, j;
3325   int rv;
3326   u8 twice_nat = 0;
3327   snat_address_t *addresses = sm->addresses;
3328
3329   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3330     {
3331       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3332         goto match;
3333     }
3334
3335   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3336     {
3337       twice_nat = 1;
3338       addresses = sm->twice_nat_addresses;
3339       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3340         goto match;
3341     }
3342
3343   return;
3344
3345 match:
3346   if (!is_delete)
3347     {
3348       /* Don't trip over lease renewal, static config */
3349       for (j = 0; j < vec_len (addresses); j++)
3350         if (addresses[j].addr.as_u32 == address->as_u32)
3351           return;
3352
3353       (void) snat_add_address (sm, address, ~0, twice_nat);
3354       /* Scan static map resolution vector */
3355       for (j = 0; j < vec_len (sm->to_resolve); j++)
3356         {
3357           rp = sm->to_resolve + j;
3358           if (rp->addr_only)
3359             continue;
3360           /* On this interface? */
3361           if (rp->sw_if_index == sw_if_index)
3362             {
3363               /* Indetity mapping? */
3364               if (rp->l_addr.as_u32 == 0)
3365                 l_addr.as_u32 = address[0].as_u32;
3366               else
3367                 l_addr.as_u32 = rp->l_addr.as_u32;
3368               /* Add the static mapping */
3369               rv = snat_add_static_mapping (l_addr,
3370                                             address[0],
3371                                             rp->l_port,
3372                                             rp->e_port,
3373                                             rp->vrf_id,
3374                                             rp->addr_only,
3375                                             ~0 /* sw_if_index */ ,
3376                                             rp->proto,
3377                                             rp->is_add, rp->twice_nat,
3378                                             rp->out2in_only, rp->tag,
3379                                             rp->identity_nat);
3380               if (rv)
3381                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3382             }
3383         }
3384       return;
3385     }
3386   else
3387     {
3388       (void) snat_del_address (sm, address[0], 1, twice_nat);
3389       return;
3390     }
3391 }
3392
3393
3394 int
3395 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3396                             u8 twice_nat)
3397 {
3398   ip4_main_t *ip4_main = sm->ip4_main;
3399   ip4_address_t *first_int_addr;
3400   snat_static_map_resolve_t *rp;
3401   u32 *indices_to_delete = 0;
3402   int i, j;
3403   u32 *auto_add_sw_if_indices =
3404     twice_nat ? sm->
3405     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3406
3407   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3408     );
3409
3410   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3411     {
3412       if (auto_add_sw_if_indices[i] == sw_if_index)
3413         {
3414           if (is_del)
3415             {
3416               /* if have address remove it */
3417               if (first_int_addr)
3418                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3419               else
3420                 {
3421                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3422                     {
3423                       rp = sm->to_resolve + j;
3424                       if (rp->sw_if_index == sw_if_index)
3425                         vec_add1 (indices_to_delete, j);
3426                     }
3427                   if (vec_len (indices_to_delete))
3428                     {
3429                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3430                         vec_del1 (sm->to_resolve, j);
3431                       vec_free (indices_to_delete);
3432                     }
3433                 }
3434               if (twice_nat)
3435                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3436               else
3437                 vec_del1 (sm->auto_add_sw_if_indices, i);
3438             }
3439           else
3440             return VNET_API_ERROR_VALUE_EXIST;
3441
3442           return 0;
3443         }
3444     }
3445
3446   if (is_del)
3447     return VNET_API_ERROR_NO_SUCH_ENTRY;
3448
3449   /* add to the auto-address list */
3450   if (twice_nat)
3451     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3452   else
3453     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3454
3455   /* If the address is already bound - or static - add it now */
3456   if (first_int_addr)
3457     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3458
3459   return 0;
3460 }
3461
3462 int
3463 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3464                    snat_protocol_t proto, u32 vrf_id, int is_in)
3465 {
3466   snat_main_per_thread_data_t *tsm;
3467   clib_bihash_kv_8_8_t kv, value;
3468   ip4_header_t ip;
3469   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3470   snat_session_key_t key;
3471   snat_session_t *s;
3472   clib_bihash_8_8_t *t;
3473
3474   if (sm->endpoint_dependent)
3475     return VNET_API_ERROR_UNSUPPORTED;
3476
3477   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3478   if (sm->num_workers > 1)
3479     tsm =
3480       vec_elt_at_index (sm->per_thread_data,
3481                         sm->worker_in2out_cb (&ip, fib_index));
3482   else
3483     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3484
3485   key.addr.as_u32 = addr->as_u32;
3486   key.port = clib_host_to_net_u16 (port);
3487   key.protocol = proto;
3488   key.fib_index = fib_index;
3489   kv.key = key.as_u64;
3490   t = is_in ? &tsm->in2out : &tsm->out2in;
3491   if (!clib_bihash_search_8_8 (t, &kv, &value))
3492     {
3493       if (pool_is_free_index (tsm->sessions, value.value))
3494         return VNET_API_ERROR_UNSPECIFIED;
3495
3496       s = pool_elt_at_index (tsm->sessions, value.value);
3497       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3498       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3499       return 0;
3500     }
3501
3502   return VNET_API_ERROR_NO_SUCH_ENTRY;
3503 }
3504
3505 int
3506 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3507                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3508                       u32 vrf_id, int is_in)
3509 {
3510   ip4_header_t ip;
3511   clib_bihash_16_8_t *t;
3512   nat_ed_ses_key_t key;
3513   clib_bihash_kv_16_8_t kv, value;
3514   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3515   snat_session_t *s;
3516   snat_main_per_thread_data_t *tsm;
3517
3518   if (!sm->endpoint_dependent)
3519     return VNET_API_ERROR_FEATURE_DISABLED;
3520
3521   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3522   if (sm->num_workers > 1)
3523     tsm =
3524       vec_elt_at_index (sm->per_thread_data,
3525                         sm->worker_in2out_cb (&ip, fib_index));
3526   else
3527     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3528
3529   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3530   key.l_addr.as_u32 = addr->as_u32;
3531   key.r_addr.as_u32 = eh_addr->as_u32;
3532   key.l_port = clib_host_to_net_u16 (port);
3533   key.r_port = clib_host_to_net_u16 (eh_port);
3534   key.proto = proto;
3535   key.fib_index = fib_index;
3536   kv.key[0] = key.as_u64[0];
3537   kv.key[1] = key.as_u64[1];
3538   if (clib_bihash_search_16_8 (t, &kv, &value))
3539     return VNET_API_ERROR_NO_SUCH_ENTRY;
3540
3541   if (pool_is_free_index (tsm->sessions, value.value))
3542     return VNET_API_ERROR_UNSPECIFIED;
3543   s = pool_elt_at_index (tsm->sessions, value.value);
3544   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3545   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3546   return 0;
3547 }
3548
3549 void
3550 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3551 {
3552   snat_main_t *sm = &snat_main;
3553
3554   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3555   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3556   sm->psid = psid;
3557   sm->psid_offset = psid_offset;
3558   sm->psid_length = psid_length;
3559 }
3560
3561 void
3562 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3563 {
3564   snat_main_t *sm = &snat_main;
3565
3566   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3567   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3568   sm->start_port = start_port;
3569   sm->end_port = end_port;
3570 }
3571
3572 void
3573 nat_set_alloc_addr_and_port_default (void)
3574 {
3575   snat_main_t *sm = &snat_main;
3576
3577   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3578   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3579 }
3580
3581 /*
3582  * fd.io coding-style-patch-verification: ON
3583  *
3584  * Local Variables:
3585  * eval: (c-set-style "gnu")
3586  * End:
3587  */