src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35
  36 #include <vpp/app/version.h>
  37
  38 snat_main_t snat_main;
  39
  40 /* *INDENT-OFF* */
  41
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat44-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  47 };
  48 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat44-out2in",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-dhcp-client-detect"),
  53 };
  54 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-classify",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-in2out",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-out2in",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-det-classify",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  74 };
  75 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-ed-in2out",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-ed-out2in",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  84                                "ip4-dhcp-client-detect"),
  85 };
  86 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-ed-classify",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-in2out-worker-handoff",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  95 };
  96 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-out2in-worker-handoff",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-handoff-classify",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-in2out-fast",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 111 };
 112 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-out2in-fast",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 116                                "ip4-dhcp-client-detect"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-hairpin-dst",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ed-hairpin-dst",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 127 };
 128
 129 /* Hook up output features */
 130 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 131   .arc_name = "ip4-output",
 132   .node_name = "nat44-in2out-output",
 133   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 136   .arc_name = "ip4-output",
 137   .node_name = "nat44-in2out-output-worker-handoff",
 138   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 141   .arc_name = "ip4-output",
 142   .node_name = "nat44-hairpin-src",
 143   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 144 };
 145 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-ed-in2out-output",
 148   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 149 };
 150 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-ed-hairpin-src",
 153   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 154 };
 155
 156 /* Hook up ip4-local features */
 157 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 158 {
 159   .arc_name = "ip4-local",
 160   .node_name = "nat44-hairpinning",
 161   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 164 {
 165   .arc_name = "ip4-local",
 166   .node_name = "nat44-ed-hairpinning",
 167   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 168 };
 169
 170
 171 VLIB_PLUGIN_REGISTER () = {
 172     .version = VPP_BUILD_VER,
 173     .description = "Network Address Translation",
 174 };
 175 /* *INDENT-ON* */
 176
 177 void
 178 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 179 {
 180   snat_session_key_t key;
 181   clib_bihash_kv_8_8_t kv;
 182   nat_ed_ses_key_t ed_key;
 183   clib_bihash_kv_16_8_t ed_kv;
 184   snat_main_per_thread_data_t *tsm =
 185     vec_elt_at_index (sm->per_thread_data, thread_index);
 186
 187   if (is_fwd_bypass_session (s))
 188     {
 189       ed_key.l_addr = s->in2out.addr;
 190       ed_key.r_addr = s->ext_host_addr;
 191       ed_key.l_port = s->in2out.port;
 192       ed_key.r_port = s->ext_host_port;
 193       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 194       ed_key.fib_index = 0;
 195       ed_kv.key[0] = ed_key.as_u64[0];
 196       ed_kv.key[1] = ed_key.as_u64[1];
 197       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 198         nat_log_warn ("in2out_ed key del failed");
 199       return;
 200     }
 201
 202   /* session lookup tables */
 203   if (is_ed_session (s))
 204     {
 205       if (is_affinity_sessions (s))
 206         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 207                              s->in2out.protocol, s->out2in.port);
 208       ed_key.l_addr = s->out2in.addr;
 209       ed_key.r_addr = s->ext_host_addr;
 210       ed_key.fib_index = s->out2in.fib_index;
 211       if (snat_is_unk_proto_session (s))
 212         {
 213           ed_key.proto = s->in2out.port;
 214           ed_key.r_port = 0;
 215           ed_key.l_port = 0;
 216         }
 217       else
 218         {
 219           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 220           ed_key.l_port = s->out2in.port;
 221           ed_key.r_port = s->ext_host_port;
 222         }
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 226         nat_log_warn ("out2in_ed key del failed");
 227       ed_key.l_addr = s->in2out.addr;
 228       ed_key.fib_index = s->in2out.fib_index;
 229       if (!snat_is_unk_proto_session (s))
 230         ed_key.l_port = s->in2out.port;
 231       if (is_twice_nat_session (s))
 232         {
 233           ed_key.r_addr = s->ext_host_nat_addr;
 234           ed_key.r_port = s->ext_host_nat_port;
 235         }
 236       ed_kv.key[0] = ed_key.as_u64[0];
 237       ed_kv.key[1] = ed_key.as_u64[1];
 238       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 239         nat_log_warn ("in2out_ed key del failed");
 240
 241       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 242                              &s->in2out.addr, s->in2out.port,
 243                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 244                              &s->out2in.addr, s->out2in.port,
 245                              &s->ext_host_addr, s->ext_host_port,
 246                              s->in2out.protocol, is_twice_nat_session (s));
 247     }
 248   else
 249     {
 250       kv.key = s->in2out.as_u64;
 251       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 252         nat_log_warn ("in2out key del failed");
 253       kv.key = s->out2in.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 255         nat_log_warn ("out2in key del failed");
 256
 257       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 258                                &s->in2out.addr, s->in2out.port,
 259                                &s->out2in.addr, s->out2in.port,
 260                                s->in2out.protocol);
 261     }
 262
 263   if (snat_is_unk_proto_session (s))
 264     return;
 265
 266   /* log NAT event */
 267   snat_ipfix_logging_nat44_ses_delete (thread_index,
 268                                        s->in2out.addr.as_u32,
 269                                        s->out2in.addr.as_u32,
 270                                        s->in2out.protocol,
 271                                        s->in2out.port,
 272                                        s->out2in.port, s->in2out.fib_index);
 273
 274   /* Twice NAT address and port for external host */
 275   if (is_twice_nat_session (s))
 276     {
 277       key.protocol = s->in2out.protocol;
 278       key.port = s->ext_host_nat_port;
 279       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 280       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 281                                           thread_index, &key);
 282     }
 283
 284   if (snat_is_session_static (s))
 285     return;
 286
 287   snat_free_outside_address_and_port (sm->addresses, thread_index,
 288                                       &s->out2in);
 289 }
 290
 291 snat_user_t *
 292 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 293                         u32 thread_index)
 294 {
 295   snat_user_t *u = 0;
 296   snat_user_key_t user_key;
 297   clib_bihash_kv_8_8_t kv, value;
 298   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 299   dlist_elt_t *per_user_list_head_elt;
 300
 301   user_key.addr.as_u32 = addr->as_u32;
 302   user_key.fib_index = fib_index;
 303   kv.key = user_key.as_u64;
 304
 305   /* Ever heard of the "user" = src ip4 address before? */
 306   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 307     {
 308       /* no, make a new one */
 309       pool_get (tsm->users, u);
 310       clib_memset (u, 0, sizeof (*u));
 311       u->addr.as_u32 = addr->as_u32;
 312       u->fib_index = fib_index;
 313
 314       pool_get (tsm->list_pool, per_user_list_head_elt);
 315
 316       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 317         tsm->list_pool;
 318
 319       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 320
 321       kv.value = u - tsm->users;
 322
 323       /* add user */
 324       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 325         nat_log_warn ("user_hash keay add failed");
 326
 327       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 328                                pool_elts (tsm->users));
 329     }
 330   else
 331     {
 332       u = pool_elt_at_index (tsm->users, value.value);
 333     }
 334
 335   return u;
 336 }
 337
 338 snat_session_t *
 339 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 340                               u32 thread_index)
 341 {
 342   snat_session_t *s;
 343   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 344   u32 oldest_per_user_translation_list_index, session_index;
 345   dlist_elt_t *oldest_per_user_translation_list_elt;
 346   dlist_elt_t *per_user_translation_list_elt;
 347
 348   /* Over quota? Recycle the least recently used translation */
 349   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 350     {
 351       oldest_per_user_translation_list_index =
 352         clib_dlist_remove_head (tsm->list_pool,
 353                                 u->sessions_per_user_list_head_index);
 354
 355       ASSERT (oldest_per_user_translation_list_index != ~0);
 356
 357       /* Add it back to the end of the LRU list */
 358       clib_dlist_addtail (tsm->list_pool,
 359                           u->sessions_per_user_list_head_index,
 360                           oldest_per_user_translation_list_index);
 361       /* Get the list element */
 362       oldest_per_user_translation_list_elt =
 363         pool_elt_at_index (tsm->list_pool,
 364                            oldest_per_user_translation_list_index);
 365
 366       /* Get the session index from the list element */
 367       session_index = oldest_per_user_translation_list_elt->value;
 368
 369       /* Get the session */
 370       s = pool_elt_at_index (tsm->sessions, session_index);
 371       nat_free_session_data (sm, s, thread_index);
 372       if (snat_is_session_static (s))
 373         u->nstaticsessions--;
 374       else
 375         u->nsessions--;
 376       s->flags = 0;
 377       s->total_bytes = 0;
 378       s->total_pkts = 0;
 379       s->state = 0;
 380       s->ext_host_addr.as_u32 = 0;
 381       s->ext_host_port = 0;
 382       s->ext_host_nat_addr.as_u32 = 0;
 383       s->ext_host_nat_port = 0;
 384     }
 385   else
 386     {
 387       pool_get (tsm->sessions, s);
 388       clib_memset (s, 0, sizeof (*s));
 389
 390       /* Create list elts */
 391       pool_get (tsm->list_pool, per_user_translation_list_elt);
 392       clib_dlist_init (tsm->list_pool,
 393                        per_user_translation_list_elt - tsm->list_pool);
 394
 395       per_user_translation_list_elt->value = s - tsm->sessions;
 396       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 397       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 398
 399       clib_dlist_addtail (tsm->list_pool,
 400                           s->per_user_list_head_index,
 401                           per_user_translation_list_elt - tsm->list_pool);
 402
 403       s->user_index = u - tsm->users;
 404       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 405                                pool_elts (tsm->sessions));
 406     }
 407
 408   return s;
 409 }
 410
 411 snat_session_t *
 412 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 413                       f64 now)
 414 {
 415   snat_session_t *s;
 416   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 417   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 418   u32 oldest_index;
 419   u64 sess_timeout_time;
 420
 421   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 422     goto alloc_new;
 423
 424   oldest_index =
 425     clib_dlist_remove_head (tsm->list_pool,
 426                             u->sessions_per_user_list_head_index);
 427   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 428   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 429   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 430   if (now >= sess_timeout_time)
 431     {
 432       clib_dlist_addtail (tsm->list_pool,
 433                           u->sessions_per_user_list_head_index, oldest_index);
 434       nat_free_session_data (sm, s, thread_index);
 435       if (snat_is_session_static (s))
 436         u->nstaticsessions--;
 437       else
 438         u->nsessions--;
 439       s->flags = 0;
 440       s->total_bytes = 0;
 441       s->total_pkts = 0;
 442       s->state = 0;
 443       s->ext_host_addr.as_u32 = 0;
 444       s->ext_host_port = 0;
 445       s->ext_host_nat_addr.as_u32 = 0;
 446       s->ext_host_nat_port = 0;
 447     }
 448   else
 449     {
 450       clib_dlist_addhead (tsm->list_pool,
 451                           u->sessions_per_user_list_head_index, oldest_index);
 452       if ((u->nsessions + u->nstaticsessions) >=
 453           sm->max_translations_per_user)
 454         {
 455           nat_log_warn ("max translations per user %U", format_ip4_address,
 456                         &u->addr);
 457           snat_ipfix_logging_max_entries_per_user
 458             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 459           return 0;
 460         }
 461       else
 462         {
 463         alloc_new:
 464           pool_get (tsm->sessions, s);
 465           clib_memset (s, 0, sizeof (*s));
 466
 467           /* Create list elts */
 468           pool_get (tsm->list_pool, per_user_translation_list_elt);
 469           clib_dlist_init (tsm->list_pool,
 470                            per_user_translation_list_elt - tsm->list_pool);
 471
 472           per_user_translation_list_elt->value = s - tsm->sessions;
 473           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 474           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 475
 476           clib_dlist_addtail (tsm->list_pool,
 477                               s->per_user_list_head_index,
 478                               per_user_translation_list_elt - tsm->list_pool);
 479         }
 480
 481       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 482                                pool_elts (tsm->sessions));
 483     }
 484
 485   return s;
 486 }
 487
 488 void
 489 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 490                           int is_add)
 491 {
 492   fib_prefix_t prefix = {
 493     .fp_len = p_len,
 494     .fp_proto = FIB_PROTOCOL_IP4,
 495     .fp_addr = {
 496                 .ip4.as_u32 = addr->as_u32,
 497                 },
 498   };
 499   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 500
 501   if (is_add)
 502     fib_table_entry_update_one_path (fib_index,
 503                                      &prefix,
 504                                      FIB_SOURCE_PLUGIN_LOW,
 505                                      (FIB_ENTRY_FLAG_CONNECTED |
 506                                       FIB_ENTRY_FLAG_LOCAL |
 507                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 508                                      DPO_PROTO_IP4,
 509                                      NULL,
 510                                      sw_if_index,
 511                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 512   else
 513     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 514 }
 515
 516 int
 517 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 518                   u8 twice_nat)
 519 {
 520   snat_address_t *ap;
 521   snat_interface_t *i;
 522   vlib_thread_main_t *tm = vlib_get_thread_main ();
 523
 524   if (twice_nat && !sm->endpoint_dependent)
 525     return VNET_API_ERROR_FEATURE_DISABLED;
 526
 527   /* Check if address already exists */
 528   /* *INDENT-OFF* */
 529   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 530     {
 531       if (ap->addr.as_u32 == addr->as_u32)
 532         return VNET_API_ERROR_VALUE_EXIST;
 533     }
 534   /* *INDENT-ON* */
 535
 536   if (twice_nat)
 537     vec_add2 (sm->twice_nat_addresses, ap, 1);
 538   else
 539     vec_add2 (sm->addresses, ap, 1);
 540
 541   ap->addr = *addr;
 542   if (vrf_id != ~0)
 543     ap->fib_index =
 544       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 545                                          FIB_SOURCE_PLUGIN_LOW);
 546   else
 547     ap->fib_index = ~0;
 548 #define _(N, i, n, s) \
 549   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 550   ap->busy_##n##_ports = 0; \
 551   ap->busy_##n##_ports_per_thread = 0;\
 552   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 553   foreach_snat_protocol
 554 #undef _
 555     if (twice_nat)
 556     return 0;
 557
 558   /* Add external address to FIB */
 559   /* *INDENT-OFF* */
 560   pool_foreach (i, sm->interfaces,
 561   ({
 562     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 563       continue;
 564
 565     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 566     break;
 567   }));
 568   pool_foreach (i, sm->output_feature_interfaces,
 569   ({
 570     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 571       continue;
 572
 573     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 574     break;
 575   }));
 576   /* *INDENT-ON* */
 577
 578   return 0;
 579 }
 580
 581 static int
 582 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 583 {
 584   snat_static_mapping_t *m;
 585   /* *INDENT-OFF* */
 586   pool_foreach (m, sm->static_mappings,
 587   ({
 588       if (is_addr_only_static_mapping (m) ||
 589           is_out2in_only_static_mapping (m) ||
 590           is_identity_static_mapping (m))
 591         continue;
 592       if (m->external_addr.as_u32 == addr.as_u32)
 593         return 1;
 594   }));
 595   /* *INDENT-ON* */
 596
 597   return 0;
 598 }
 599
 600 void
 601 increment_v4_address (ip4_address_t * a)
 602 {
 603   u32 v;
 604
 605   v = clib_net_to_host_u32 (a->as_u32) + 1;
 606   a->as_u32 = clib_host_to_net_u32 (v);
 607 }
 608
 609 static void
 610 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 611                                        ip4_address_t l_addr,
 612                                        u16 l_port,
 613                                        u32 sw_if_index,
 614                                        u16 e_port,
 615                                        u32 vrf_id,
 616                                        snat_protocol_t proto,
 617                                        int addr_only, int is_add, u8 * tag,
 618                                        int twice_nat, int out2in_only,
 619                                        int identity_nat)
 620 {
 621   snat_static_map_resolve_t *rp;
 622
 623   vec_add2 (sm->to_resolve, rp, 1);
 624   rp->l_addr.as_u32 = l_addr.as_u32;
 625   rp->l_port = l_port;
 626   rp->sw_if_index = sw_if_index;
 627   rp->e_port = e_port;
 628   rp->vrf_id = vrf_id;
 629   rp->proto = proto;
 630   rp->addr_only = addr_only;
 631   rp->is_add = is_add;
 632   rp->twice_nat = twice_nat;
 633   rp->out2in_only = out2in_only;
 634   rp->identity_nat = identity_nat;
 635   rp->tag = vec_dup (tag);
 636 }
 637
 638 static u32
 639 get_thread_idx_by_port (u16 e_port)
 640 {
 641   snat_main_t *sm = &snat_main;
 642   u32 thread_idx = sm->num_workers;
 643   if (sm->num_workers > 1)
 644     {
 645       thread_idx =
 646         sm->first_worker_index +
 647         sm->workers[(e_port - 1024) / sm->port_per_thread];
 648     }
 649   return thread_idx;
 650 }
 651
 652 int
 653 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 654                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 655                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 656                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 657                          u8 identity_nat)
 658 {
 659   snat_main_t *sm = &snat_main;
 660   snat_static_mapping_t *m;
 661   snat_session_key_t m_key;
 662   clib_bihash_kv_8_8_t kv, value;
 663   snat_address_t *a = 0;
 664   u32 fib_index = ~0;
 665   snat_interface_t *interface;
 666   int i;
 667   snat_main_per_thread_data_t *tsm;
 668   snat_user_key_t u_key;
 669   snat_user_t *u;
 670   dlist_elt_t *head, *elt;
 671   u32 elt_index, head_index;
 672   u32 ses_index;
 673   u64 user_index;
 674   snat_session_t *s;
 675   snat_static_map_resolve_t *rp, *rp_match = 0;
 676   nat44_lb_addr_port_t *local;
 677   u32 find = ~0;
 678
 679   if (!sm->endpoint_dependent)
 680     {
 681       if (twice_nat || out2in_only)
 682         return VNET_API_ERROR_FEATURE_DISABLED;
 683     }
 684
 685   /* If the external address is a specific interface address */
 686   if (sw_if_index != ~0)
 687     {
 688       ip4_address_t *first_int_addr;
 689
 690       for (i = 0; i < vec_len (sm->to_resolve); i++)
 691         {
 692           rp = sm->to_resolve + i;
 693           if (rp->sw_if_index != sw_if_index ||
 694               rp->l_addr.as_u32 != l_addr.as_u32 ||
 695               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 696             continue;
 697
 698           if (!addr_only)
 699             {
 700               if (rp->l_port != l_port || rp->e_port != e_port
 701                   || rp->proto != proto)
 702                 continue;
 703             }
 704
 705           rp_match = rp;
 706           break;
 707         }
 708
 709       /* Might be already set... */
 710       first_int_addr = ip4_interface_first_address
 711         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 712
 713       if (is_add)
 714         {
 715           if (rp_match)
 716             return VNET_API_ERROR_VALUE_EXIST;
 717
 718           snat_add_static_mapping_when_resolved
 719             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 720              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 721
 722           /* DHCP resolution required? */
 723           if (first_int_addr == 0)
 724             {
 725               return 0;
 726             }
 727           else
 728             {
 729               e_addr.as_u32 = first_int_addr->as_u32;
 730               /* Identity mapping? */
 731               if (l_addr.as_u32 == 0)
 732                 l_addr.as_u32 = e_addr.as_u32;
 733             }
 734         }
 735       else
 736         {
 737           if (!rp_match)
 738             return VNET_API_ERROR_NO_SUCH_ENTRY;
 739
 740           vec_del1 (sm->to_resolve, i);
 741
 742           if (first_int_addr)
 743             {
 744               e_addr.as_u32 = first_int_addr->as_u32;
 745               /* Identity mapping? */
 746               if (l_addr.as_u32 == 0)
 747                 l_addr.as_u32 = e_addr.as_u32;
 748             }
 749           else
 750             return 0;
 751         }
 752     }
 753
 754   m_key.addr = e_addr;
 755   m_key.port = addr_only ? 0 : e_port;
 756   m_key.protocol = addr_only ? 0 : proto;
 757   m_key.fib_index = 0;
 758   kv.key = m_key.as_u64;
 759   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 760     m = 0;
 761   else
 762     m = pool_elt_at_index (sm->static_mappings, value.value);
 763
 764   if (is_add)
 765     {
 766       if (m)
 767         {
 768           if (is_identity_static_mapping (m))
 769             {
 770               /* *INDENT-OFF* */
 771               pool_foreach (local, m->locals,
 772               ({
 773                 if (local->vrf_id == vrf_id)
 774                   return VNET_API_ERROR_VALUE_EXIST;
 775               }));
 776               /* *INDENT-ON* */
 777               pool_get (m->locals, local);
 778               local->vrf_id = vrf_id;
 779               local->fib_index =
 780                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 781                                                    FIB_SOURCE_PLUGIN_LOW);
 782               m_key.addr = m->local_addr;
 783               m_key.port = m->local_port;
 784               m_key.protocol = m->proto;
 785               m_key.fib_index = local->fib_index;
 786               kv.key = m_key.as_u64;
 787               kv.value = m - sm->static_mappings;
 788               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 789               return 0;
 790             }
 791           else
 792             return VNET_API_ERROR_VALUE_EXIST;
 793         }
 794
 795       if (twice_nat && addr_only)
 796         return VNET_API_ERROR_UNSUPPORTED;
 797
 798       /* Convert VRF id to FIB index */
 799       if (vrf_id != ~0)
 800         fib_index =
 801           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 802                                              FIB_SOURCE_PLUGIN_LOW);
 803       /* If not specified use inside VRF id from SNAT plugin startup config */
 804       else
 805         {
 806           fib_index = sm->inside_fib_index;
 807           vrf_id = sm->inside_vrf_id;
 808         }
 809
 810       if (!(out2in_only || identity_nat))
 811         {
 812           m_key.addr = l_addr;
 813           m_key.port = addr_only ? 0 : l_port;
 814           m_key.protocol = addr_only ? 0 : proto;
 815           m_key.fib_index = fib_index;
 816           kv.key = m_key.as_u64;
 817           if (!clib_bihash_search_8_8
 818               (&sm->static_mapping_by_local, &kv, &value))
 819             return VNET_API_ERROR_VALUE_EXIST;
 820         }
 821
 822       /* Find external address in allocated addresses and reserve port for
 823          address and port pair mapping when dynamic translations enabled */
 824       if (!(addr_only || sm->static_mapping_only || out2in_only))
 825         {
 826           for (i = 0; i < vec_len (sm->addresses); i++)
 827             {
 828               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 829                 {
 830                   a = sm->addresses + i;
 831                   /* External port must be unused */
 832                   switch (proto)
 833                     {
 834 #define _(N, j, n, s) \
 835                     case SNAT_PROTOCOL_##N: \
 836                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 837                         return VNET_API_ERROR_INVALID_VALUE; \
 838                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 839                       if (e_port > 1024) \
 840                         { \
 841                           a->busy_##n##_ports++; \
 842                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 843                         } \
 844                       break;
 845                       foreach_snat_protocol
 846 #undef _
 847                     default:
 848                       nat_log_info ("unknown protocol");
 849                       return VNET_API_ERROR_INVALID_VALUE_2;
 850                     }
 851                   break;
 852                 }
 853             }
 854           /* External address must be allocated */
 855           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 856             {
 857               if (sw_if_index != ~0)
 858                 {
 859                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 860                     {
 861                       rp = sm->to_resolve + i;
 862                       if (rp->addr_only)
 863                         continue;
 864                       if (rp->sw_if_index != sw_if_index &&
 865                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 866                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 867                           rp->e_port != e_port && rp->proto != proto)
 868                         continue;
 869
 870                       vec_del1 (sm->to_resolve, i);
 871                       break;
 872                     }
 873                 }
 874               return VNET_API_ERROR_NO_SUCH_ENTRY;
 875             }
 876         }
 877
 878       pool_get (sm->static_mappings, m);
 879       clib_memset (m, 0, sizeof (*m));
 880       m->tag = vec_dup (tag);
 881       m->local_addr = l_addr;
 882       m->external_addr = e_addr;
 883       m->twice_nat = twice_nat;
 884       if (out2in_only)
 885         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 886       if (addr_only)
 887         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 888       if (identity_nat)
 889         {
 890           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 891           pool_get (m->locals, local);
 892           local->vrf_id = vrf_id;
 893           local->fib_index = fib_index;
 894         }
 895       else
 896         {
 897           m->vrf_id = vrf_id;
 898           m->fib_index = fib_index;
 899         }
 900       if (!addr_only)
 901         {
 902           m->local_port = l_port;
 903           m->external_port = e_port;
 904           m->proto = proto;
 905         }
 906
 907       if (sm->num_workers > 1)
 908         {
 909           ip4_header_t ip = {
 910             .src_address = m->local_addr,
 911           };
 912           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 913           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 914         }
 915       else
 916         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 917
 918       m_key.addr = m->local_addr;
 919       m_key.port = m->local_port;
 920       m_key.protocol = m->proto;
 921       m_key.fib_index = fib_index;
 922       kv.key = m_key.as_u64;
 923       kv.value = m - sm->static_mappings;
 924       if (!out2in_only)
 925         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 926
 927       m_key.addr = m->external_addr;
 928       m_key.port = m->external_port;
 929       m_key.fib_index = 0;
 930       kv.key = m_key.as_u64;
 931       kv.value = m - sm->static_mappings;
 932       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 933
 934       /* Delete dynamic sessions matching local address (+ local port) */
 935       if (!(sm->static_mapping_only))
 936         {
 937           u_key.addr = m->local_addr;
 938           u_key.fib_index = m->fib_index;
 939           kv.key = u_key.as_u64;
 940           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 941             {
 942               user_index = value.value;
 943               u = pool_elt_at_index (tsm->users, user_index);
 944               if (u->nsessions)
 945                 {
 946                   head_index = u->sessions_per_user_list_head_index;
 947                   head = pool_elt_at_index (tsm->list_pool, head_index);
 948                   elt_index = head->next;
 949                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 950                   ses_index = elt->value;
 951                   while (ses_index != ~0)
 952                     {
 953                       s = pool_elt_at_index (tsm->sessions, ses_index);
 954                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 955                       ses_index = elt->value;
 956
 957                       if (snat_is_session_static (s))
 958                         continue;
 959
 960                       if (!addr_only
 961                           && (clib_net_to_host_u16 (s->in2out.port) !=
 962                               m->local_port))
 963                         continue;
 964
 965                       nat_free_session_data (sm, s,
 966                                              tsm - sm->per_thread_data);
 967                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 968
 969                       if (!addr_only && !sm->endpoint_dependent)
 970                         break;
 971                     }
 972                 }
 973             }
 974         }
 975     }
 976   else
 977     {
 978       if (!m)
 979         {
 980           if (sw_if_index != ~0)
 981             return 0;
 982           else
 983             return VNET_API_ERROR_NO_SUCH_ENTRY;
 984         }
 985
 986       if (identity_nat)
 987         {
 988           if (vrf_id == ~0)
 989             vrf_id = sm->inside_vrf_id;
 990
 991           /* *INDENT-OFF* */
 992           pool_foreach (local, m->locals,
 993           ({
 994             if (local->vrf_id == vrf_id)
 995               find = local - m->locals;
 996           }));
 997           /* *INDENT-ON* */
 998           if (find == ~0)
 999             return VNET_API_ERROR_NO_SUCH_ENTRY;
1000
1001           local = pool_elt_at_index (m->locals, find);
1002           fib_index = local->fib_index;
1003           pool_put (m->locals, local);
1004         }
1005       else
1006         fib_index = m->fib_index;
1007
1008       /* Free external address port */
1009       if (!(addr_only || sm->static_mapping_only || out2in_only))
1010         {
1011           for (i = 0; i < vec_len (sm->addresses); i++)
1012             {
1013               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1014                 {
1015                   a = sm->addresses + i;
1016                   switch (proto)
1017                     {
1018 #define _(N, j, n, s) \
1019                     case SNAT_PROTOCOL_##N: \
1020                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1021                       if (e_port > 1024) \
1022                         { \
1023                           a->busy_##n##_ports--; \
1024                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1025                         } \
1026                       break;
1027                       foreach_snat_protocol
1028 #undef _
1029                     default:
1030                       nat_log_info ("unknown protocol");
1031                       return VNET_API_ERROR_INVALID_VALUE_2;
1032                     }
1033                   break;
1034                 }
1035             }
1036         }
1037
1038       if (sm->num_workers > 1)
1039         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1040       else
1041         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1042
1043       m_key.addr = m->local_addr;
1044       m_key.port = m->local_port;
1045       m_key.protocol = m->proto;
1046       m_key.fib_index = fib_index;
1047       kv.key = m_key.as_u64;
1048       if (!out2in_only)
1049         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1050
1051       /* Delete session(s) for static mapping if exist */
1052       if (!(sm->static_mapping_only) ||
1053           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1054         {
1055           u_key.addr = m->local_addr;
1056           u_key.fib_index = fib_index;
1057           kv.key = u_key.as_u64;
1058           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1059             {
1060               user_index = value.value;
1061               u = pool_elt_at_index (tsm->users, user_index);
1062               if (u->nstaticsessions)
1063                 {
1064                   head_index = u->sessions_per_user_list_head_index;
1065                   head = pool_elt_at_index (tsm->list_pool, head_index);
1066                   elt_index = head->next;
1067                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1068                   ses_index = elt->value;
1069                   while (ses_index != ~0)
1070                     {
1071                       s = pool_elt_at_index (tsm->sessions, ses_index);
1072                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1073                       ses_index = elt->value;
1074
1075                       if (!addr_only)
1076                         {
1077                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1078                               (clib_net_to_host_u16 (s->out2in.port) !=
1079                                e_port))
1080                             continue;
1081                         }
1082
1083                       if (is_lb_session (s))
1084                         continue;
1085
1086                       if (!snat_is_session_static (s))
1087                         continue;
1088
1089                       nat_free_session_data (sm, s,
1090                                              tsm - sm->per_thread_data);
1091                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1092
1093                       if (!addr_only && !sm->endpoint_dependent)
1094                         break;
1095                     }
1096                 }
1097             }
1098         }
1099
1100       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1101       if (pool_elts (m->locals))
1102         return 0;
1103
1104       m_key.addr = m->external_addr;
1105       m_key.port = m->external_port;
1106       m_key.fib_index = 0;
1107       kv.key = m_key.as_u64;
1108       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1109
1110       vec_free (m->tag);
1111       vec_free (m->workers);
1112       /* Delete static mapping from pool */
1113       pool_put (sm->static_mappings, m);
1114     }
1115
1116   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1117     return 0;
1118
1119   /* Add/delete external address to FIB */
1120   /* *INDENT-OFF* */
1121   pool_foreach (interface, sm->interfaces,
1122   ({
1123     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1124       continue;
1125
1126     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1127     break;
1128   }));
1129   pool_foreach (interface, sm->output_feature_interfaces,
1130   ({
1131     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1132       continue;
1133
1134     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1135     break;
1136   }));
1137   /* *INDENT-ON* */
1138
1139   return 0;
1140 }
1141
1142 int
1143 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1144                                  snat_protocol_t proto,
1145                                  nat44_lb_addr_port_t * locals, u8 is_add,
1146                                  twice_nat_type_t twice_nat, u8 out2in_only,
1147                                  u8 * tag, u32 affinity)
1148 {
1149   snat_main_t *sm = &snat_main;
1150   snat_static_mapping_t *m;
1151   snat_session_key_t m_key;
1152   clib_bihash_kv_8_8_t kv, value;
1153   snat_address_t *a = 0;
1154   int i;
1155   nat44_lb_addr_port_t *local;
1156   u32 elt_index, head_index, ses_index;
1157   snat_main_per_thread_data_t *tsm;
1158   snat_user_key_t u_key;
1159   snat_user_t *u;
1160   snat_session_t *s;
1161   dlist_elt_t *head, *elt;
1162   uword *bitmap = 0;
1163
1164   if (!sm->endpoint_dependent)
1165     return VNET_API_ERROR_FEATURE_DISABLED;
1166
1167   m_key.addr = e_addr;
1168   m_key.port = e_port;
1169   m_key.protocol = proto;
1170   m_key.fib_index = 0;
1171   kv.key = m_key.as_u64;
1172   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1173     m = 0;
1174   else
1175     m = pool_elt_at_index (sm->static_mappings, value.value);
1176
1177   if (is_add)
1178     {
1179       if (m)
1180         return VNET_API_ERROR_VALUE_EXIST;
1181
1182       if (vec_len (locals) < 2)
1183         return VNET_API_ERROR_INVALID_VALUE;
1184
1185       /* Find external address in allocated addresses and reserve port for
1186          address and port pair mapping when dynamic translations enabled */
1187       if (!(sm->static_mapping_only || out2in_only))
1188         {
1189           for (i = 0; i < vec_len (sm->addresses); i++)
1190             {
1191               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1192                 {
1193                   a = sm->addresses + i;
1194                   /* External port must be unused */
1195                   switch (proto)
1196                     {
1197 #define _(N, j, n, s) \
1198                     case SNAT_PROTOCOL_##N: \
1199                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1200                         return VNET_API_ERROR_INVALID_VALUE; \
1201                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1202                       if (e_port > 1024) \
1203                         { \
1204                           a->busy_##n##_ports++; \
1205                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1206                         } \
1207                       break;
1208                       foreach_snat_protocol
1209 #undef _
1210                     default:
1211                       nat_log_info ("unknown protocol");
1212                       return VNET_API_ERROR_INVALID_VALUE_2;
1213                     }
1214                   break;
1215                 }
1216             }
1217           /* External address must be allocated */
1218           if (!a)
1219             return VNET_API_ERROR_NO_SUCH_ENTRY;
1220         }
1221
1222       pool_get (sm->static_mappings, m);
1223       clib_memset (m, 0, sizeof (*m));
1224       m->tag = vec_dup (tag);
1225       m->external_addr = e_addr;
1226       m->external_port = e_port;
1227       m->proto = proto;
1228       m->twice_nat = twice_nat;
1229       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1230       if (out2in_only)
1231         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1232       m->affinity = affinity;
1233
1234       if (affinity)
1235         m->affinity_per_service_list_head_index =
1236           nat_affinity_get_per_service_list_head_index ();
1237       else
1238         m->affinity_per_service_list_head_index = ~0;
1239
1240       m_key.addr = m->external_addr;
1241       m_key.port = m->external_port;
1242       m_key.protocol = m->proto;
1243       m_key.fib_index = 0;
1244       kv.key = m_key.as_u64;
1245       kv.value = m - sm->static_mappings;
1246       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1247         {
1248           nat_log_err ("static_mapping_by_external key add failed");
1249           return VNET_API_ERROR_UNSPECIFIED;
1250         }
1251
1252       m_key.fib_index = m->fib_index;
1253       for (i = 0; i < vec_len (locals); i++)
1254         {
1255           locals[i].fib_index =
1256             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1257                                                locals[i].vrf_id,
1258                                                FIB_SOURCE_PLUGIN_LOW);
1259           m_key.addr = locals[i].addr;
1260           m_key.fib_index = locals[i].fib_index;
1261           if (!out2in_only)
1262             {
1263               m_key.port = locals[i].port;
1264               kv.key = m_key.as_u64;
1265               kv.value = m - sm->static_mappings;
1266               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1267             }
1268           locals[i].prefix = (i == 0) ? locals[i].probability :
1269             (locals[i - 1].prefix + locals[i].probability);
1270           pool_get (m->locals, local);
1271           *local = locals[i];
1272           if (sm->num_workers > 1)
1273             {
1274               ip4_header_t ip = {
1275                 .src_address = locals[i].addr,
1276               };
1277               bitmap =
1278                 clib_bitmap_set (bitmap,
1279                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1280             }
1281         }
1282
1283       /* Assign workers */
1284       if (sm->num_workers > 1)
1285         {
1286           /* *INDENT-OFF* */
1287           clib_bitmap_foreach (i, bitmap,
1288             ({
1289                vec_add1(m->workers, i);
1290             }));
1291           /* *INDENT-ON* */
1292         }
1293     }
1294   else
1295     {
1296       if (!m)
1297         return VNET_API_ERROR_NO_SUCH_ENTRY;
1298
1299       if (!is_lb_static_mapping (m))
1300         return VNET_API_ERROR_INVALID_VALUE;
1301
1302       /* Free external address port */
1303       if (!(sm->static_mapping_only || out2in_only))
1304         {
1305           for (i = 0; i < vec_len (sm->addresses); i++)
1306             {
1307               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1308                 {
1309                   a = sm->addresses + i;
1310                   switch (proto)
1311                     {
1312 #define _(N, j, n, s) \
1313                     case SNAT_PROTOCOL_##N: \
1314                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1315                       if (e_port > 1024) \
1316                         { \
1317                           a->busy_##n##_ports--; \
1318                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1319                         } \
1320                       break;
1321                       foreach_snat_protocol
1322 #undef _
1323                     default:
1324                       nat_log_info ("unknown protocol");
1325                       return VNET_API_ERROR_INVALID_VALUE_2;
1326                     }
1327                   break;
1328                 }
1329             }
1330         }
1331
1332       m_key.addr = m->external_addr;
1333       m_key.port = m->external_port;
1334       m_key.protocol = m->proto;
1335       m_key.fib_index = 0;
1336       kv.key = m_key.as_u64;
1337       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1338         {
1339           nat_log_err ("static_mapping_by_external key del failed");
1340           return VNET_API_ERROR_UNSPECIFIED;
1341         }
1342
1343       /* *INDENT-OFF* */
1344       pool_foreach (local, m->locals,
1345       ({
1346           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1347                             FIB_SOURCE_PLUGIN_LOW);
1348           m_key.addr = local->addr;
1349           if (!out2in_only)
1350             {
1351               m_key.port = local->port;
1352               m_key.fib_index = local->fib_index;
1353               kv.key = m_key.as_u64;
1354               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1355                 {
1356                   nat_log_err ("static_mapping_by_local key del failed");
1357                   return VNET_API_ERROR_UNSPECIFIED;
1358                 }
1359             }
1360
1361           if (sm->num_workers > 1)
1362             {
1363               ip4_header_t ip = {
1364                 .src_address = local->addr,
1365               };
1366               tsm = vec_elt_at_index (sm->per_thread_data,
1367                                       sm->worker_in2out_cb (&ip, m->fib_index));
1368             }
1369           else
1370             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1371
1372           /* Delete sessions */
1373           u_key.addr = local->addr;
1374           u_key.fib_index = local->fib_index;
1375           kv.key = u_key.as_u64;
1376           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1377             {
1378               u = pool_elt_at_index (tsm->users, value.value);
1379               if (u->nstaticsessions)
1380                 {
1381                   head_index = u->sessions_per_user_list_head_index;
1382                   head = pool_elt_at_index (tsm->list_pool, head_index);
1383                   elt_index = head->next;
1384                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1385                   ses_index = elt->value;
1386                   while (ses_index != ~0)
1387                     {
1388                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1389                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1390                       ses_index = elt->value;
1391
1392                       if (!(is_lb_session (s)))
1393                         continue;
1394
1395                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1396                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1397                         continue;
1398
1399                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1400                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1401                     }
1402                 }
1403             }
1404       }));
1405       /* *INDENT-ON* */
1406       if (m->affinity)
1407         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1408       pool_free (m->locals);
1409       vec_free (m->tag);
1410       vec_free (m->workers);
1411
1412       pool_put (sm->static_mappings, m);
1413     }
1414
1415   return 0;
1416 }
1417
1418 int
1419 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1420                                        ip4_address_t l_addr, u16 l_port,
1421                                        snat_protocol_t proto, u32 vrf_id,
1422                                        u8 probability, u8 is_add)
1423 {
1424   snat_main_t *sm = &snat_main;
1425   snat_static_mapping_t *m = 0;
1426   snat_session_key_t m_key;
1427   clib_bihash_kv_8_8_t kv, value;
1428   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1429   snat_main_per_thread_data_t *tsm;
1430   snat_user_key_t u_key;
1431   snat_user_t *u;
1432   snat_session_t *s;
1433   dlist_elt_t *head, *elt;
1434   u32 elt_index, head_index, ses_index, *locals = 0;
1435   uword *bitmap = 0;
1436   int i;
1437
1438   if (!sm->endpoint_dependent)
1439     return VNET_API_ERROR_FEATURE_DISABLED;
1440
1441   m_key.addr = e_addr;
1442   m_key.port = e_port;
1443   m_key.protocol = proto;
1444   m_key.fib_index = 0;
1445   kv.key = m_key.as_u64;
1446   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1447     m = pool_elt_at_index (sm->static_mappings, value.value);
1448
1449   if (!m)
1450     return VNET_API_ERROR_NO_SUCH_ENTRY;
1451
1452   if (!is_lb_static_mapping (m))
1453     return VNET_API_ERROR_INVALID_VALUE;
1454
1455   /* *INDENT-OFF* */
1456   pool_foreach (local, m->locals,
1457   ({
1458     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1459         (local->vrf_id == vrf_id))
1460       {
1461         match_local = local;
1462         break;
1463       }
1464   }));
1465   /* *INDENT-ON* */
1466
1467   if (is_add)
1468     {
1469       if (match_local)
1470         return VNET_API_ERROR_VALUE_EXIST;
1471
1472       pool_get (m->locals, local);
1473       clib_memset (local, 0, sizeof (*local));
1474       local->addr.as_u32 = l_addr.as_u32;
1475       local->port = l_port;
1476       local->probability = probability;
1477       local->vrf_id = vrf_id;
1478       local->fib_index =
1479         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1480                                            FIB_SOURCE_PLUGIN_LOW);
1481
1482       if (!is_out2in_only_static_mapping (m))
1483         {
1484           m_key.addr = l_addr;
1485           m_key.port = l_port;
1486           m_key.fib_index = local->fib_index;
1487           kv.key = m_key.as_u64;
1488           kv.value = m - sm->static_mappings;
1489           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1490             nat_log_err ("static_mapping_by_local key add failed");
1491         }
1492     }
1493   else
1494     {
1495       if (!match_local)
1496         return VNET_API_ERROR_NO_SUCH_ENTRY;
1497
1498       if (pool_elts (m->locals) < 3)
1499         return VNET_API_ERROR_UNSPECIFIED;
1500
1501       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1502                         FIB_SOURCE_PLUGIN_LOW);
1503
1504       if (!is_out2in_only_static_mapping (m))
1505         {
1506           m_key.addr = l_addr;
1507           m_key.port = l_port;
1508           m_key.fib_index = match_local->fib_index;
1509           kv.key = m_key.as_u64;
1510           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1511             nat_log_err ("static_mapping_by_local key del failed");
1512         }
1513
1514       if (sm->num_workers > 1)
1515         {
1516           ip4_header_t ip = {
1517             .src_address = local->addr,
1518           };
1519           tsm = vec_elt_at_index (sm->per_thread_data,
1520                                   sm->worker_in2out_cb (&ip, m->fib_index));
1521         }
1522       else
1523         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1524
1525       /* Delete sessions */
1526       u_key.addr = match_local->addr;
1527       u_key.fib_index = match_local->fib_index;
1528       kv.key = u_key.as_u64;
1529       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1530         {
1531           u = pool_elt_at_index (tsm->users, value.value);
1532           if (u->nstaticsessions)
1533             {
1534               head_index = u->sessions_per_user_list_head_index;
1535               head = pool_elt_at_index (tsm->list_pool, head_index);
1536               elt_index = head->next;
1537               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1538               ses_index = elt->value;
1539               while (ses_index != ~0)
1540                 {
1541                   s = pool_elt_at_index (tsm->sessions, ses_index);
1542                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1543                   ses_index = elt->value;
1544
1545                   if (!(is_lb_session (s)))
1546                     continue;
1547
1548                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1549                       (clib_net_to_host_u16 (s->in2out.port) !=
1550                        match_local->port))
1551                     continue;
1552
1553                   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1554                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1555                 }
1556             }
1557         }
1558
1559       pool_put (m->locals, match_local);
1560     }
1561
1562   vec_free (m->workers);
1563
1564   /* *INDENT-OFF* */
1565   pool_foreach (local, m->locals,
1566   ({
1567     vec_add1 (locals, local - m->locals);
1568     if (sm->num_workers > 1)
1569       {
1570         ip4_header_t ip;
1571         ip.src_address.as_u32 = local->addr.as_u32,
1572         bitmap = clib_bitmap_set (bitmap,
1573                                   sm->worker_in2out_cb (&ip, local->fib_index),
1574                                   1);
1575       }
1576   }));
1577   /* *INDENT-ON* */
1578
1579   ASSERT (vec_len (locals) > 1);
1580
1581   local = pool_elt_at_index (m->locals, locals[0]);
1582   local->prefix = local->probability;
1583   for (i = 1; i < vec_len (locals); i++)
1584     {
1585       local = pool_elt_at_index (m->locals, locals[i]);
1586       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1587       local->prefix = local->probability + prev_local->prefix;
1588     }
1589
1590   /* Assign workers */
1591   if (sm->num_workers > 1)
1592     {
1593       /* *INDENT-OFF* */
1594       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1595       /* *INDENT-ON* */
1596     }
1597
1598   return 0;
1599 }
1600
1601 int
1602 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1603                   u8 twice_nat)
1604 {
1605   snat_address_t *a = 0;
1606   snat_session_t *ses;
1607   u32 *ses_to_be_removed = 0, *ses_index;
1608   snat_main_per_thread_data_t *tsm;
1609   snat_static_mapping_t *m;
1610   snat_interface_t *interface;
1611   int i;
1612   snat_address_t *addresses =
1613     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1614
1615   /* Find SNAT address */
1616   for (i = 0; i < vec_len (addresses); i++)
1617     {
1618       if (addresses[i].addr.as_u32 == addr.as_u32)
1619         {
1620           a = addresses + i;
1621           break;
1622         }
1623     }
1624   if (!a)
1625     return VNET_API_ERROR_NO_SUCH_ENTRY;
1626
1627   if (delete_sm)
1628     {
1629       /* *INDENT-OFF* */
1630       pool_foreach (m, sm->static_mappings,
1631       ({
1632           if (m->external_addr.as_u32 == addr.as_u32)
1633             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1634                                             m->local_port, m->external_port,
1635                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1636                                             m->proto, 0, m->twice_nat,
1637                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1638       }));
1639       /* *INDENT-ON* */
1640     }
1641   else
1642     {
1643       /* Check if address is used in some static mapping */
1644       if (is_snat_address_used_in_static_mapping (sm, addr))
1645         {
1646           nat_log_notice ("address used in static mapping");
1647           return VNET_API_ERROR_UNSPECIFIED;
1648         }
1649     }
1650
1651   if (a->fib_index != ~0)
1652     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1653
1654   /* Delete sessions using address */
1655   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1656     {
1657       /* *INDENT-OFF* */
1658       vec_foreach (tsm, sm->per_thread_data)
1659         {
1660           pool_foreach (ses, tsm->sessions, ({
1661             if (ses->out2in.addr.as_u32 == addr.as_u32)
1662               {
1663                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1664                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1665               }
1666           }));
1667
1668           vec_foreach (ses_index, ses_to_be_removed)
1669             {
1670               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1671               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1672             }
1673
1674           vec_free (ses_to_be_removed);
1675         }
1676       /* *INDENT-ON* */
1677     }
1678
1679 #define _(N, i, n, s) \
1680   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1681   vec_free (a->busy_##n##_ports_per_thread);
1682   foreach_snat_protocol
1683 #undef _
1684     if (twice_nat)
1685     {
1686       vec_del1 (sm->twice_nat_addresses, i);
1687       return 0;
1688     }
1689   else
1690     vec_del1 (sm->addresses, i);
1691
1692   /* Delete external address from FIB */
1693   /* *INDENT-OFF* */
1694   pool_foreach (interface, sm->interfaces,
1695   ({
1696     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1697       continue;
1698
1699     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1700     break;
1701   }));
1702   pool_foreach (interface, sm->output_feature_interfaces,
1703   ({
1704     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1705       continue;
1706
1707     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1708     break;
1709   }));
1710   /* *INDENT-ON* */
1711
1712   return 0;
1713 }
1714
1715 int
1716 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1717 {
1718   snat_main_t *sm = &snat_main;
1719   snat_interface_t *i;
1720   const char *feature_name, *del_feature_name;
1721   snat_address_t *ap;
1722   snat_static_mapping_t *m;
1723   snat_det_map_t *dm;
1724   nat_outside_fib_t *outside_fib;
1725   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1726                                                        sw_if_index);
1727
1728   if (sm->out2in_dpo && !is_inside)
1729     return VNET_API_ERROR_UNSUPPORTED;
1730
1731   /* *INDENT-OFF* */
1732   pool_foreach (i, sm->output_feature_interfaces,
1733   ({
1734     if (i->sw_if_index == sw_if_index)
1735       return VNET_API_ERROR_VALUE_EXIST;
1736   }));
1737   /* *INDENT-ON* */
1738
1739   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1740     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1741   else
1742     {
1743       if (sm->num_workers > 1 && !sm->deterministic)
1744         feature_name =
1745           is_inside ? "nat44-in2out-worker-handoff" :
1746           "nat44-out2in-worker-handoff";
1747       else if (sm->deterministic)
1748         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1749       else if (sm->endpoint_dependent)
1750         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1751       else
1752         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1753     }
1754
1755   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1756     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1757                                                       NAT_FQ_NELTS);
1758
1759   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1760     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1761                                                       NAT_FQ_NELTS);
1762
1763   if (!is_inside)
1764     {
1765       /* *INDENT-OFF* */
1766       vec_foreach (outside_fib, sm->outside_fibs)
1767         {
1768           if (outside_fib->fib_index == fib_index)
1769             {
1770               if (is_del)
1771                 {
1772                   outside_fib->refcount--;
1773                   if (!outside_fib->refcount)
1774                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1775                 }
1776               else
1777                 outside_fib->refcount++;
1778               goto feature_set;
1779             }
1780         }
1781       /* *INDENT-ON* */
1782       if (!is_del)
1783         {
1784           vec_add2 (sm->outside_fibs, outside_fib, 1);
1785           outside_fib->refcount = 1;
1786           outside_fib->fib_index = fib_index;
1787         }
1788     }
1789 feature_set:
1790   /* *INDENT-OFF* */
1791   pool_foreach (i, sm->interfaces,
1792   ({
1793     if (i->sw_if_index == sw_if_index)
1794       {
1795         if (is_del)
1796           {
1797             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1798               {
1799                 if (is_inside)
1800                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1801                 else
1802                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1803
1804                 if (sm->num_workers > 1 && !sm->deterministic)
1805                   {
1806                     del_feature_name = "nat44-handoff-classify";
1807                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1808                                                  "nat44-out2in-worker-handoff";
1809                   }
1810                 else if (sm->deterministic)
1811                   {
1812                     del_feature_name = "nat44-det-classify";
1813                     feature_name = !is_inside ?  "nat44-det-in2out" :
1814                                                  "nat44-det-out2in";
1815                   }
1816                 else if (sm->endpoint_dependent)
1817                   {
1818                     del_feature_name = "nat44-ed-classify";
1819                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1820                                                  "nat44-ed-out2in";
1821                   }
1822                 else
1823                   {
1824                     del_feature_name = "nat44-classify";
1825                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1826                   }
1827
1828                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1829                                              sw_if_index, 0, 0, 0);
1830                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1831                                              sw_if_index, 1, 0, 0);
1832                 if (!is_inside)
1833                   {
1834                     if (sm->endpoint_dependent)
1835                       vnet_feature_enable_disable ("ip4-local",
1836                                                    "nat44-ed-hairpinning",
1837                                                    sw_if_index, 1, 0, 0);
1838                     else if (!sm->deterministic)
1839                       vnet_feature_enable_disable ("ip4-local",
1840                                                    "nat44-hairpinning",
1841                                                    sw_if_index, 1, 0, 0);
1842                   }
1843               }
1844             else
1845               {
1846                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1847                                              sw_if_index, 0, 0, 0);
1848                 pool_put (sm->interfaces, i);
1849                 if (is_inside)
1850                   {
1851                     if (sm->endpoint_dependent)
1852                       vnet_feature_enable_disable ("ip4-local",
1853                                                    "nat44-ed-hairpinning",
1854                                                    sw_if_index, 0, 0, 0);
1855                     else if (!sm->deterministic)
1856                       vnet_feature_enable_disable ("ip4-local",
1857                                                    "nat44-hairpinning",
1858                                                    sw_if_index, 0, 0, 0);
1859                   }
1860               }
1861           }
1862         else
1863           {
1864             if ((nat_interface_is_inside(i) && is_inside) ||
1865                 (nat_interface_is_outside(i) && !is_inside))
1866               return 0;
1867
1868             if (sm->num_workers > 1 && !sm->deterministic)
1869               {
1870                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1871                                                  "nat44-out2in-worker-handoff";
1872                 feature_name = "nat44-handoff-classify";
1873               }
1874             else if (sm->deterministic)
1875               {
1876                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1877                                                  "nat44-det-out2in";
1878                 feature_name = "nat44-det-classify";
1879               }
1880             else if (sm->endpoint_dependent)
1881               {
1882                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1883                                                  "nat44-ed-out2in";
1884                 feature_name = "nat44-ed-classify";
1885               }
1886             else
1887               {
1888                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1889                 feature_name = "nat44-classify";
1890               }
1891
1892             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1893                                          sw_if_index, 0, 0, 0);
1894             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1895                                          sw_if_index, 1, 0, 0);
1896             if (!is_inside)
1897               {
1898                 if (sm->endpoint_dependent)
1899                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1900                                                sw_if_index, 0, 0, 0);
1901                 else if (!sm->deterministic)
1902                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1903                                                sw_if_index, 0, 0, 0);
1904               }
1905             goto set_flags;
1906           }
1907
1908         goto fib;
1909       }
1910   }));
1911   /* *INDENT-ON* */
1912
1913   if (is_del)
1914     return VNET_API_ERROR_NO_SUCH_ENTRY;
1915
1916   pool_get (sm->interfaces, i);
1917   i->sw_if_index = sw_if_index;
1918   i->flags = 0;
1919   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1920                                0);
1921
1922   if (is_inside && !sm->out2in_dpo)
1923     {
1924       if (sm->endpoint_dependent)
1925         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1926                                      sw_if_index, 1, 0, 0);
1927       else if (!sm->deterministic)
1928         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1929                                      sw_if_index, 1, 0, 0);
1930     }
1931
1932 set_flags:
1933   if (is_inside)
1934     {
1935       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1936       return 0;
1937     }
1938   else
1939     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1940
1941   /* Add/delete external addresses to FIB */
1942 fib:
1943   /* *INDENT-OFF* */
1944   vec_foreach (ap, sm->addresses)
1945     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1946
1947   pool_foreach (m, sm->static_mappings,
1948   ({
1949     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1950       continue;
1951
1952     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1953   }));
1954
1955   pool_foreach (dm, sm->det_maps,
1956   ({
1957     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1958   }));
1959   /* *INDENT-ON* */
1960
1961   return 0;
1962 }
1963
1964 int
1965 snat_interface_add_del_output_feature (u32 sw_if_index,
1966                                        u8 is_inside, int is_del)
1967 {
1968   snat_main_t *sm = &snat_main;
1969   snat_interface_t *i;
1970   snat_address_t *ap;
1971   snat_static_mapping_t *m;
1972   nat_outside_fib_t *outside_fib;
1973   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1974                                                        sw_if_index);
1975
1976
1977   if (sm->deterministic ||
1978       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1979     return VNET_API_ERROR_UNSUPPORTED;
1980
1981   /* *INDENT-OFF* */
1982   pool_foreach (i, sm->interfaces,
1983   ({
1984     if (i->sw_if_index == sw_if_index)
1985       return VNET_API_ERROR_VALUE_EXIST;
1986   }));
1987   /* *INDENT-ON* */
1988
1989   if (!is_inside)
1990     {
1991       /* *INDENT-OFF* */
1992       vec_foreach (outside_fib, sm->outside_fibs)
1993         {
1994           if (outside_fib->fib_index == fib_index)
1995             {
1996               if (is_del)
1997                 {
1998                   outside_fib->refcount--;
1999                   if (!outside_fib->refcount)
2000                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2001                 }
2002               else
2003                 outside_fib->refcount++;
2004               goto feature_set;
2005             }
2006         }
2007       /* *INDENT-ON* */
2008       if (!is_del)
2009         {
2010           vec_add2 (sm->outside_fibs, outside_fib, 1);
2011           outside_fib->refcount = 1;
2012           outside_fib->fib_index = fib_index;
2013         }
2014     }
2015
2016 feature_set:
2017   if (is_inside)
2018     {
2019       if (sm->endpoint_dependent)
2020         {
2021           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2022                                        sw_if_index, !is_del, 0, 0);
2023           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2024                                        sw_if_index, !is_del, 0, 0);
2025         }
2026       else
2027         {
2028           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2029                                        sw_if_index, !is_del, 0, 0);
2030           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2031                                        sw_if_index, !is_del, 0, 0);
2032         }
2033       goto fq;
2034     }
2035
2036   if (sm->num_workers > 1)
2037     {
2038       vnet_feature_enable_disable ("ip4-unicast",
2039                                    "nat44-out2in-worker-handoff",
2040                                    sw_if_index, !is_del, 0, 0);
2041       vnet_feature_enable_disable ("ip4-output",
2042                                    "nat44-in2out-output-worker-handoff",
2043                                    sw_if_index, !is_del, 0, 0);
2044     }
2045   else
2046     {
2047       if (sm->endpoint_dependent)
2048         {
2049           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2050                                        sw_if_index, !is_del, 0, 0);
2051           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2052                                        sw_if_index, !is_del, 0, 0);
2053         }
2054       else
2055         {
2056           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2057                                        sw_if_index, !is_del, 0, 0);
2058           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2059                                        sw_if_index, !is_del, 0, 0);
2060         }
2061     }
2062
2063 fq:
2064   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2065     sm->fq_in2out_output_index =
2066       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2067
2068   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2069     sm->fq_out2in_index =
2070       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2071
2072   /* *INDENT-OFF* */
2073   pool_foreach (i, sm->output_feature_interfaces,
2074   ({
2075     if (i->sw_if_index == sw_if_index)
2076       {
2077         if (is_del)
2078           pool_put (sm->output_feature_interfaces, i);
2079         else
2080           return VNET_API_ERROR_VALUE_EXIST;
2081
2082         goto fib;
2083       }
2084   }));
2085   /* *INDENT-ON* */
2086
2087   if (is_del)
2088     return VNET_API_ERROR_NO_SUCH_ENTRY;
2089
2090   pool_get (sm->output_feature_interfaces, i);
2091   i->sw_if_index = sw_if_index;
2092   i->flags = 0;
2093   if (is_inside)
2094     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2095   else
2096     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2097
2098   /* Add/delete external addresses to FIB */
2099 fib:
2100   if (is_inside)
2101     return 0;
2102
2103   /* *INDENT-OFF* */
2104   vec_foreach (ap, sm->addresses)
2105     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2106
2107   pool_foreach (m, sm->static_mappings,
2108   ({
2109     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2110       continue;
2111
2112     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2113   }));
2114   /* *INDENT-ON* */
2115
2116   return 0;
2117 }
2118
2119 int
2120 snat_set_workers (uword * bitmap)
2121 {
2122   snat_main_t *sm = &snat_main;
2123   int i, j = 0;
2124
2125   if (sm->num_workers < 2)
2126     return VNET_API_ERROR_FEATURE_DISABLED;
2127
2128   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2129     return VNET_API_ERROR_INVALID_WORKER;
2130
2131   vec_free (sm->workers);
2132   /* *INDENT-OFF* */
2133   clib_bitmap_foreach (i, bitmap,
2134     ({
2135       vec_add1(sm->workers, i);
2136       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2137       j++;
2138     }));
2139   /* *INDENT-ON* */
2140
2141   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2142   sm->num_snat_thread = _vec_len (sm->workers);
2143
2144   return 0;
2145 }
2146
2147 static void
2148 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2149                          u32 old_fib_index)
2150 {
2151   snat_main_t *sm = &snat_main;
2152   nat_outside_fib_t *outside_fib;
2153   snat_interface_t *i;
2154   u8 is_add = 1;
2155
2156   if (new_fib_index == old_fib_index)
2157     return;
2158
2159   if (!vec_len (sm->outside_fibs))
2160     return;
2161
2162   pool_foreach (i, sm->interfaces, (
2163                                      {
2164                                      if (i->sw_if_index == sw_if_index)
2165                                      {
2166                                      if (!(nat_interface_is_outside (i)))
2167                                      return;}
2168                                      }
2169                 ));
2170   vec_foreach (outside_fib, sm->outside_fibs)
2171   {
2172     if (outside_fib->fib_index == old_fib_index)
2173       {
2174         outside_fib->refcount--;
2175         if (!outside_fib->refcount)
2176           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2177         break;
2178       }
2179   }
2180
2181   vec_foreach (outside_fib, sm->outside_fibs)
2182   {
2183     if (outside_fib->fib_index == new_fib_index)
2184       {
2185         outside_fib->refcount++;
2186         is_add = 0;
2187         break;
2188       }
2189   }
2190
2191   if (is_add)
2192     {
2193       vec_add2 (sm->outside_fibs, outside_fib, 1);
2194       outside_fib->refcount = 1;
2195       outside_fib->fib_index = new_fib_index;
2196     }
2197 }
2198
2199 static void
2200 snat_ip4_table_bind (ip4_main_t * im,
2201                      uword opaque,
2202                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2203 {
2204   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2205 }
2206
2207 static void
2208 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2209                                        uword opaque,
2210                                        u32 sw_if_index,
2211                                        ip4_address_t * address,
2212                                        u32 address_length,
2213                                        u32 if_address_index, u32 is_delete);
2214
2215 static void
2216 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2217                                  uword opaque,
2218                                  u32 sw_if_index,
2219                                  ip4_address_t * address,
2220                                  u32 address_length,
2221                                  u32 if_address_index, u32 is_delete);
2222
2223 static int
2224 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2225                                  u32 fib_index,
2226                                  u32 thread_index,
2227                                  snat_session_key_t * k,
2228                                  u16 port_per_thread, u32 snat_thread_index);
2229
2230 static clib_error_t *
2231 snat_init (vlib_main_t * vm)
2232 {
2233   snat_main_t *sm = &snat_main;
2234   clib_error_t *error = 0;
2235   ip4_main_t *im = &ip4_main;
2236   ip_lookup_main_t *lm = &im->lookup_main;
2237   uword *p;
2238   vlib_thread_registration_t *tr;
2239   vlib_thread_main_t *tm = vlib_get_thread_main ();
2240   uword *bitmap = 0;
2241   u32 i;
2242   ip4_add_del_interface_address_callback_t cb4;
2243   vlib_node_t *node;
2244
2245   sm->vlib_main = vm;
2246   sm->vnet_main = vnet_get_main ();
2247   sm->ip4_main = im;
2248   sm->ip4_lookup_main = lm;
2249   sm->api_main = &api_main;
2250   sm->first_worker_index = 0;
2251   sm->num_workers = 0;
2252   sm->num_snat_thread = 1;
2253   sm->workers = 0;
2254   sm->port_per_thread = 0xffff - 1024;
2255   sm->fq_in2out_index = ~0;
2256   sm->fq_out2in_index = ~0;
2257   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2258   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2259   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2260   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2261   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2262   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2263   sm->forwarding_enabled = 0;
2264   sm->log_class = vlib_log_register_class ("nat", 0);
2265   sm->mss_clamping = 0;
2266
2267   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2268   sm->error_node_index = node->index;
2269
2270   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2271   sm->in2out_node_index = node->index;
2272   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2273   sm->in2out_output_node_index = node->index;
2274   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2275   sm->in2out_fast_node_index = node->index;
2276   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2277   sm->in2out_slowpath_node_index = node->index;
2278   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2279   sm->in2out_slowpath_output_node_index = node->index;
2280   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-reass");
2281   sm->in2out_reass_node_index = node->index;
2282
2283   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2284   sm->ed_in2out_node_index = node->index;
2285   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2286   sm->ed_in2out_slowpath_node_index = node->index;
2287   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-reass");
2288   sm->ed_in2out_reass_node_index = node->index;
2289
2290   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2291   sm->out2in_node_index = node->index;
2292   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2293   sm->out2in_fast_node_index = node->index;
2294   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-reass");
2295   sm->out2in_reass_node_index = node->index;
2296
2297   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2298   sm->ed_out2in_node_index = node->index;
2299   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2300   sm->ed_out2in_slowpath_node_index = node->index;
2301   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-reass");
2302   sm->ed_out2in_reass_node_index = node->index;
2303
2304   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2305   sm->det_in2out_node_index = node->index;
2306   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2307   sm->det_out2in_node_index = node->index;
2308
2309   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2310   sm->hairpinning_node_index = node->index;
2311   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2312   sm->hairpin_dst_node_index = node->index;
2313   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2314   sm->hairpin_src_node_index = node->index;
2315   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2316   sm->ed_hairpinning_node_index = node->index;
2317   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2318   sm->ed_hairpin_dst_node_index = node->index;
2319   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2320   sm->ed_hairpin_src_node_index = node->index;
2321
2322   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2323   if (p)
2324     {
2325       tr = (vlib_thread_registration_t *) p[0];
2326       if (tr)
2327         {
2328           sm->num_workers = tr->count;
2329           sm->first_worker_index = tr->first_index;
2330         }
2331     }
2332
2333   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2334
2335   /* Use all available workers by default */
2336   if (sm->num_workers > 1)
2337     {
2338       for (i = 0; i < sm->num_workers; i++)
2339         bitmap = clib_bitmap_set (bitmap, i, 1);
2340       snat_set_workers (bitmap);
2341       clib_bitmap_free (bitmap);
2342     }
2343   else
2344     {
2345       sm->per_thread_data[0].snat_thread_index = 0;
2346     }
2347
2348   error = snat_api_init (vm, sm);
2349   if (error)
2350     return error;
2351
2352   /* Set up the interface address add/del callback */
2353   cb4.function = snat_ip4_add_del_interface_address_cb;
2354   cb4.function_opaque = 0;
2355
2356   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2357
2358   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2359   cb4.function_opaque = 0;
2360
2361   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2362
2363   nat_dpo_module_init ();
2364
2365   /* Init counters */
2366   sm->total_users.name = "total-users";
2367   sm->total_users.stat_segment_name = "/nat44/total-users";
2368   vlib_validate_simple_counter (&sm->total_users, 0);
2369   vlib_zero_simple_counter (&sm->total_users, 0);
2370   sm->total_sessions.name = "total-sessions";
2371   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2372   vlib_validate_simple_counter (&sm->total_sessions, 0);
2373   vlib_zero_simple_counter (&sm->total_sessions, 0);
2374
2375   /* Init IPFIX logging */
2376   snat_ipfix_logging_init (vm);
2377
2378   /* Init NAT64 */
2379   error = nat64_init (vm);
2380   if (error)
2381     return error;
2382
2383   dslite_init (vm);
2384
2385   nat66_init (vm);
2386
2387   ip4_table_bind_callback_t cbt4 = {
2388     .function = snat_ip4_table_bind,
2389   };
2390   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2391
2392   /* Init virtual fragmenentation reassembly */
2393   return nat_reass_init (vm);
2394 }
2395
2396 VLIB_INIT_FUNCTION (snat_init);
2397
2398 void
2399 snat_free_outside_address_and_port (snat_address_t * addresses,
2400                                     u32 thread_index, snat_session_key_t * k)
2401 {
2402   snat_address_t *a;
2403   u32 address_index;
2404   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2405
2406   for (address_index = 0; address_index < vec_len (addresses);
2407        address_index++)
2408     {
2409       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2410         break;
2411     }
2412
2413   ASSERT (address_index < vec_len (addresses));
2414
2415   a = addresses + address_index;
2416
2417   switch (k->protocol)
2418     {
2419 #define _(N, i, n, s) \
2420     case SNAT_PROTOCOL_##N: \
2421       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2422         port_host_byte_order) == 1); \
2423       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2424         port_host_byte_order, 0); \
2425       a->busy_##n##_ports--; \
2426       a->busy_##n##_ports_per_thread[thread_index]--; \
2427       break;
2428       foreach_snat_protocol
2429 #undef _
2430     default:
2431       nat_log_info ("unknown protocol");
2432       return;
2433     }
2434 }
2435
2436 int
2437 snat_static_mapping_match (snat_main_t * sm,
2438                            snat_session_key_t match,
2439                            snat_session_key_t * mapping,
2440                            u8 by_external,
2441                            u8 * is_addr_only,
2442                            twice_nat_type_t * twice_nat,
2443                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2444                            u8 * is_identity_nat)
2445 {
2446   clib_bihash_kv_8_8_t kv, value;
2447   snat_static_mapping_t *m;
2448   snat_session_key_t m_key;
2449   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2450   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2451   u8 backend_index;
2452   nat44_lb_addr_port_t *local;
2453
2454   m_key.fib_index = match.fib_index;
2455   if (by_external)
2456     {
2457       mapping_hash = &sm->static_mapping_by_external;
2458       m_key.fib_index = 0;
2459     }
2460
2461   m_key.addr = match.addr;
2462   m_key.port = clib_net_to_host_u16 (match.port);
2463   m_key.protocol = match.protocol;
2464
2465   kv.key = m_key.as_u64;
2466
2467   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2468     {
2469       /* Try address only mapping */
2470       m_key.port = 0;
2471       m_key.protocol = 0;
2472       kv.key = m_key.as_u64;
2473       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2474         return 1;
2475     }
2476
2477   m = pool_elt_at_index (sm->static_mappings, value.value);
2478
2479   if (by_external)
2480     {
2481       if (is_lb_static_mapping (m))
2482         {
2483           if (PREDICT_FALSE (lb != 0))
2484             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2485           if (m->affinity)
2486             {
2487               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2488                                               match.protocol, match.port,
2489                                               &backend_index))
2490                 goto get_local;
2491
2492               local = pool_elt_at_index (m->locals, backend_index);
2493               mapping->addr = local->addr;
2494               mapping->port = clib_host_to_net_u16 (local->port);
2495               mapping->fib_index = local->fib_index;
2496               goto end;
2497             }
2498         get_local:
2499           /* *INDENT-OFF* */
2500           pool_foreach_index (i, m->locals,
2501           ({
2502             vec_add1 (tmp, i);
2503           }));
2504           /* *INDENT-ON* */
2505           hi = vec_len (tmp) - 1;
2506           local = pool_elt_at_index (m->locals, tmp[hi]);
2507           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2508           while (lo < hi)
2509             {
2510               mid = ((hi - lo) >> 1) + lo;
2511               local = pool_elt_at_index (m->locals, tmp[mid]);
2512               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2513             }
2514           local = pool_elt_at_index (m->locals, tmp[lo]);
2515           if (!(local->prefix >= rand))
2516             return 1;
2517           if (PREDICT_FALSE (sm->num_workers > 1))
2518             {
2519               ip4_header_t ip = {
2520                 .src_address = local->addr,
2521               };
2522               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2523                   vlib_get_thread_index ())
2524                 goto get_local;
2525             }
2526           mapping->addr = local->addr;
2527           mapping->port = clib_host_to_net_u16 (local->port);
2528           mapping->fib_index = local->fib_index;
2529           if (m->affinity)
2530             {
2531               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2532                                                 match.protocol, match.port,
2533                                                 tmp[lo], m->affinity,
2534                                                 m->affinity_per_service_list_head_index))
2535                 nat_log_info ("create affinity record failed");
2536             }
2537           vec_free (tmp);
2538         }
2539       else
2540         {
2541           if (PREDICT_FALSE (lb != 0))
2542             *lb = NO_LB_NAT;
2543           mapping->fib_index = m->fib_index;
2544           mapping->addr = m->local_addr;
2545           /* Address only mapping doesn't change port */
2546           mapping->port = is_addr_only_static_mapping (m) ? match.port
2547             : clib_host_to_net_u16 (m->local_port);
2548         }
2549       mapping->protocol = m->proto;
2550     }
2551   else
2552     {
2553       mapping->addr = m->external_addr;
2554       /* Address only mapping doesn't change port */
2555       mapping->port = is_addr_only_static_mapping (m) ? match.port
2556         : clib_host_to_net_u16 (m->external_port);
2557       mapping->fib_index = sm->outside_fib_index;
2558     }
2559
2560 end:
2561   if (PREDICT_FALSE (is_addr_only != 0))
2562     *is_addr_only = is_addr_only_static_mapping (m);
2563
2564   if (PREDICT_FALSE (twice_nat != 0))
2565     *twice_nat = m->twice_nat;
2566
2567   if (PREDICT_FALSE (is_identity_nat != 0))
2568     *is_identity_nat = is_identity_static_mapping (m);
2569
2570   return 0;
2571 }
2572
2573 static_always_inline u16
2574 snat_random_port (u16 min, u16 max)
2575 {
2576   snat_main_t *sm = &snat_main;
2577   return min + random_u32 (&sm->random_seed) /
2578     (random_u32_max () / (max - min + 1) + 1);
2579 }
2580
2581 int
2582 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2583                                      u32 fib_index,
2584                                      u32 thread_index,
2585                                      snat_session_key_t * k,
2586                                      u16 port_per_thread,
2587                                      u32 snat_thread_index)
2588 {
2589   snat_main_t *sm = &snat_main;
2590
2591   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2592                                   port_per_thread, snat_thread_index);
2593 }
2594
2595 static int
2596 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2597                                  u32 fib_index,
2598                                  u32 thread_index,
2599                                  snat_session_key_t * k,
2600                                  u16 port_per_thread, u32 snat_thread_index)
2601 {
2602   int i;
2603   snat_address_t *a, *ga = 0;
2604   u32 portnum;
2605
2606   for (i = 0; i < vec_len (addresses); i++)
2607     {
2608       a = addresses + i;
2609       switch (k->protocol)
2610         {
2611 #define _(N, j, n, s) \
2612         case SNAT_PROTOCOL_##N: \
2613           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2614             { \
2615               if (a->fib_index == fib_index) \
2616                 { \
2617                   while (1) \
2618                     { \
2619                       portnum = (port_per_thread * \
2620                         snat_thread_index) + \
2621                         snat_random_port(1, port_per_thread) + 1024; \
2622                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2623                         continue; \
2624                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2625                       a->busy_##n##_ports_per_thread[thread_index]++; \
2626                       a->busy_##n##_ports++; \
2627                       k->addr = a->addr; \
2628                       k->port = clib_host_to_net_u16(portnum); \
2629                       return 0; \
2630                     } \
2631                 } \
2632               else if (a->fib_index == ~0) \
2633                 { \
2634                   ga = a; \
2635                 } \
2636             } \
2637           break;
2638           foreach_snat_protocol
2639 #undef _
2640         default:
2641           nat_log_info ("unknown protocol");
2642           return 1;
2643         }
2644
2645     }
2646
2647   if (ga)
2648     {
2649       a = ga;
2650       switch (k->protocol)
2651         {
2652 #define _(N, j, n, s) \
2653         case SNAT_PROTOCOL_##N: \
2654           while (1) \
2655             { \
2656               portnum = (port_per_thread * \
2657                 snat_thread_index) + \
2658                 snat_random_port(1, port_per_thread) + 1024; \
2659               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2660                 continue; \
2661               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2662               a->busy_##n##_ports_per_thread[thread_index]++; \
2663               a->busy_##n##_ports++; \
2664               k->addr = a->addr; \
2665               k->port = clib_host_to_net_u16(portnum); \
2666               return 0; \
2667             }
2668           break;
2669           foreach_snat_protocol
2670 #undef _
2671         default:
2672           nat_log_info ("unknown protocol");
2673           return 1;
2674         }
2675     }
2676
2677   /* Totally out of translations to use... */
2678   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2679   return 1;
2680 }
2681
2682 static int
2683 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2684                               u32 fib_index,
2685                               u32 thread_index,
2686                               snat_session_key_t * k,
2687                               u16 port_per_thread, u32 snat_thread_index)
2688 {
2689   snat_main_t *sm = &snat_main;
2690   snat_address_t *a = addresses;
2691   u16 m, ports, portnum, A, j;
2692   m = 16 - (sm->psid_offset + sm->psid_length);
2693   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2694
2695   if (!vec_len (addresses))
2696     goto exhausted;
2697
2698   switch (k->protocol)
2699     {
2700 #define _(N, i, n, s) \
2701     case SNAT_PROTOCOL_##N: \
2702       if (a->busy_##n##_ports < ports) \
2703         { \
2704           while (1) \
2705             { \
2706               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2707               j = snat_random_port(0, pow2_mask(m)); \
2708               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2709               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2710                 continue; \
2711               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2712               a->busy_##n##_ports++; \
2713               k->addr = a->addr; \
2714               k->port = clib_host_to_net_u16 (portnum); \
2715               return 0; \
2716             } \
2717         } \
2718       break;
2719       foreach_snat_protocol
2720 #undef _
2721     default:
2722       nat_log_info ("unknown protocol");
2723       return 1;
2724     }
2725
2726 exhausted:
2727   /* Totally out of translations to use... */
2728   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2729   return 1;
2730 }
2731
2732 static int
2733 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2734                                u32 fib_index,
2735                                u32 thread_index,
2736                                snat_session_key_t * k,
2737                                u16 port_per_thread, u32 snat_thread_index)
2738 {
2739   snat_main_t *sm = &snat_main;
2740   snat_address_t *a = addresses;
2741   u16 portnum, ports;
2742
2743   ports = sm->end_port - sm->start_port + 1;
2744
2745   if (!vec_len (addresses))
2746     goto exhausted;
2747
2748   switch (k->protocol)
2749     {
2750 #define _(N, i, n, s) \
2751     case SNAT_PROTOCOL_##N: \
2752       if (a->busy_##n##_ports < ports) \
2753         { \
2754           while (1) \
2755             { \
2756               portnum = snat_random_port(sm->start_port, sm->end_port); \
2757               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2758                 continue; \
2759               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2760               a->busy_##n##_ports++; \
2761               k->addr = a->addr; \
2762               k->port = clib_host_to_net_u16 (portnum); \
2763               return 0; \
2764             } \
2765         } \
2766       break;
2767       foreach_snat_protocol
2768 #undef _
2769     default:
2770       nat_log_info ("unknown protocol");
2771       return 1;
2772     }
2773
2774 exhausted:
2775   /* Totally out of translations to use... */
2776   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2777   return 1;
2778 }
2779
2780 void
2781 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2782 {
2783   dpo_id_t dpo_v4 = DPO_INVALID;
2784   fib_prefix_t pfx = {
2785     .fp_proto = FIB_PROTOCOL_IP4,
2786     .fp_len = 32,
2787     .fp_addr.ip4.as_u32 = addr.as_u32,
2788   };
2789
2790   if (is_add)
2791     {
2792       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2793       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2794                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2795       dpo_reset (&dpo_v4);
2796     }
2797   else
2798     {
2799       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2800     }
2801 }
2802
2803 u8 *
2804 format_session_kvp (u8 * s, va_list * args)
2805 {
2806   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2807   snat_session_key_t k;
2808
2809   k.as_u64 = v->key;
2810
2811   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2812
2813   return s;
2814 }
2815
2816 u8 *
2817 format_static_mapping_kvp (u8 * s, va_list * args)
2818 {
2819   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2820   snat_session_key_t k;
2821
2822   k.as_u64 = v->key;
2823
2824   s = format (s, "%U static-mapping-index %llu",
2825               format_static_mapping_key, &k, v->value);
2826
2827   return s;
2828 }
2829
2830 u8 *
2831 format_user_kvp (u8 * s, va_list * args)
2832 {
2833   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2834   snat_user_key_t k;
2835
2836   k.as_u64 = v->key;
2837
2838   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2839               k.fib_index, v->value);
2840
2841   return s;
2842 }
2843
2844 u8 *
2845 format_ed_session_kvp (u8 * s, va_list * args)
2846 {
2847   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2848   nat_ed_ses_key_t k;
2849
2850   k.as_u64[0] = v->key[0];
2851   k.as_u64[1] = v->key[1];
2852
2853   s =
2854     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2855             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2856             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2857             format_ip_protocol, k.proto, k.fib_index, v->value);
2858
2859   return s;
2860 }
2861
2862 static u32
2863 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2864 {
2865   snat_main_t *sm = &snat_main;
2866   u32 next_worker_index = 0;
2867   u32 hash;
2868
2869   next_worker_index = sm->first_worker_index;
2870   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2871     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2872
2873   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2874     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2875   else
2876     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2877
2878   return next_worker_index;
2879 }
2880
2881 static u32
2882 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2883 {
2884   snat_main_t *sm = &snat_main;
2885   udp_header_t *udp;
2886   u16 port;
2887   snat_session_key_t m_key;
2888   clib_bihash_kv_8_8_t kv, value;
2889   snat_static_mapping_t *m;
2890   u32 proto;
2891   u32 next_worker_index = 0;
2892
2893   /* first try static mappings without port */
2894   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2895     {
2896       m_key.addr = ip0->dst_address;
2897       m_key.port = 0;
2898       m_key.protocol = 0;
2899       m_key.fib_index = rx_fib_index0;
2900       kv.key = m_key.as_u64;
2901       if (!clib_bihash_search_8_8
2902           (&sm->static_mapping_by_external, &kv, &value))
2903         {
2904           m = pool_elt_at_index (sm->static_mappings, value.value);
2905           return m->workers[0];
2906         }
2907     }
2908
2909   proto = ip_proto_to_snat_proto (ip0->protocol);
2910   udp = ip4_next_header (ip0);
2911   port = udp->dst_port;
2912
2913   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2914     {
2915       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2916         return vlib_get_thread_index ();
2917
2918       nat_reass_ip4_t *reass;
2919       reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2920                                   ip0->fragment_id, ip0->protocol);
2921
2922       if (reass && (reass->thread_index != (u32) ~ 0))
2923         return reass->thread_index;
2924
2925       if (ip4_is_first_fragment (ip0))
2926         {
2927           reass =
2928             nat_ip4_reass_create (ip0->src_address, ip0->dst_address,
2929                                   ip0->fragment_id, ip0->protocol);
2930           if (!reass)
2931             goto no_reass;
2932
2933           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2934             {
2935               m_key.addr = ip0->dst_address;
2936               m_key.port = clib_net_to_host_u16 (port);
2937               m_key.protocol = proto;
2938               m_key.fib_index = rx_fib_index0;
2939               kv.key = m_key.as_u64;
2940               if (!clib_bihash_search_8_8
2941                   (&sm->static_mapping_by_external, &kv, &value))
2942                 {
2943                   m = pool_elt_at_index (sm->static_mappings, value.value);
2944                   reass->thread_index = m->workers[0];
2945                   return reass->thread_index;
2946                 }
2947             }
2948           reass->thread_index = sm->first_worker_index;
2949           reass->thread_index +=
2950             sm->workers[(clib_net_to_host_u16 (port) - 1024) /
2951                         sm->port_per_thread];
2952           return reass->thread_index;
2953         }
2954       else
2955         return vlib_get_thread_index ();
2956     }
2957
2958 no_reass:
2959   /* unknown protocol */
2960   if (PREDICT_FALSE (proto == ~0))
2961     {
2962       /* use current thread */
2963       return vlib_get_thread_index ();
2964     }
2965
2966   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2967     {
2968       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2969       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2970       if (!icmp_is_error_message (icmp))
2971         port = echo->identifier;
2972       else
2973         {
2974           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2975           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2976           void *l4_header = ip4_next_header (inner_ip);
2977           switch (proto)
2978             {
2979             case SNAT_PROTOCOL_ICMP:
2980               icmp = (icmp46_header_t *) l4_header;
2981               echo = (icmp_echo_header_t *) (icmp + 1);
2982               port = echo->identifier;
2983               break;
2984             case SNAT_PROTOCOL_UDP:
2985             case SNAT_PROTOCOL_TCP:
2986               port = ((tcp_udp_header_t *) l4_header)->src_port;
2987               break;
2988             default:
2989               return vlib_get_thread_index ();
2990             }
2991         }
2992     }
2993
2994   /* try static mappings with port */
2995   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2996     {
2997       m_key.addr = ip0->dst_address;
2998       m_key.port = clib_net_to_host_u16 (port);
2999       m_key.protocol = proto;
3000       m_key.fib_index = rx_fib_index0;
3001       kv.key = m_key.as_u64;
3002       if (!clib_bihash_search_8_8
3003           (&sm->static_mapping_by_external, &kv, &value))
3004         {
3005           m = pool_elt_at_index (sm->static_mappings, value.value);
3006           return m->workers[0];
3007         }
3008     }
3009
3010   /* worker by outside port */
3011   next_worker_index = sm->first_worker_index;
3012   next_worker_index +=
3013     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3014   return next_worker_index;
3015 }
3016
3017 static u32
3018 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
3019 {
3020   snat_main_t *sm = &snat_main;
3021   clib_bihash_kv_8_8_t kv, value;
3022   u32 proto, next_worker_index = 0;
3023   udp_header_t *udp;
3024   u16 port;
3025   snat_static_mapping_t *m;
3026   u32 hash;
3027
3028   /* first try static mappings without port */
3029   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3030     {
3031       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
3032       if (!clib_bihash_search_8_8
3033           (&sm->static_mapping_by_external, &kv, &value))
3034         {
3035           m = pool_elt_at_index (sm->static_mappings, value.value);
3036           return m->workers[0];
3037         }
3038     }
3039
3040   proto = ip_proto_to_snat_proto (ip->protocol);
3041
3042   /* unknown protocol */
3043   if (PREDICT_FALSE (proto == ~0))
3044     {
3045       /* use current thread */
3046       return vlib_get_thread_index ();
3047     }
3048
3049   udp = ip4_next_header (ip);
3050   port = udp->dst_port;
3051
3052   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3053     {
3054       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3055       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3056       if (!icmp_is_error_message (icmp))
3057         port = echo->identifier;
3058       else
3059         {
3060           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3061           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3062           void *l4_header = ip4_next_header (inner_ip);
3063           switch (proto)
3064             {
3065             case SNAT_PROTOCOL_ICMP:
3066               icmp = (icmp46_header_t *) l4_header;
3067               echo = (icmp_echo_header_t *) (icmp + 1);
3068               port = echo->identifier;
3069               break;
3070             case SNAT_PROTOCOL_UDP:
3071             case SNAT_PROTOCOL_TCP:
3072               port = ((tcp_udp_header_t *) l4_header)->src_port;
3073               break;
3074             default:
3075               return vlib_get_thread_index ();
3076             }
3077         }
3078     }
3079
3080   /* try static mappings with port */
3081   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3082     {
3083       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
3084                   clib_net_to_host_u16 (port));
3085       if (!clib_bihash_search_8_8
3086           (&sm->static_mapping_by_external, &kv, &value))
3087         {
3088           m = pool_elt_at_index (sm->static_mappings, value.value);
3089           if (!is_lb_static_mapping (m))
3090             return m->workers[0];
3091
3092           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3093             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3094
3095           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3096             return m->workers[hash & (_vec_len (m->workers) - 1)];
3097           else
3098             return m->workers[hash % _vec_len (m->workers)];
3099         }
3100     }
3101
3102   /* worker by outside port */
3103   next_worker_index = sm->first_worker_index;
3104   next_worker_index +=
3105     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3106
3107   return next_worker_index;
3108 }
3109
3110 static clib_error_t *
3111 snat_config (vlib_main_t * vm, unformat_input_t * input)
3112 {
3113   snat_main_t *sm = &snat_main;
3114   nat66_main_t *nm = &nat66_main;
3115   u32 translation_buckets = 1024;
3116   u32 translation_memory_size = 128 << 20;
3117   u32 user_buckets = 128;
3118   u32 user_memory_size = 64 << 20;
3119   u32 max_translations_per_user = 100;
3120   u32 outside_vrf_id = 0;
3121   u32 outside_ip6_vrf_id = 0;
3122   u32 inside_vrf_id = 0;
3123   u32 static_mapping_buckets = 1024;
3124   u32 static_mapping_memory_size = 64 << 20;
3125   u32 nat64_bib_buckets = 1024;
3126   u32 nat64_bib_memory_size = 128 << 20;
3127   u32 nat64_st_buckets = 2048;
3128   u32 nat64_st_memory_size = 256 << 20;
3129   u8 static_mapping_only = 0;
3130   u8 static_mapping_connection_tracking = 0;
3131   snat_main_per_thread_data_t *tsm;
3132   dslite_main_t *dm = &dslite_main;
3133
3134   sm->deterministic = 0;
3135   sm->out2in_dpo = 0;
3136   sm->endpoint_dependent = 0;
3137
3138   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3139     {
3140       if (unformat
3141           (input, "translation hash buckets %d", &translation_buckets))
3142         ;
3143       else if (unformat (input, "translation hash memory %d",
3144                          &translation_memory_size));
3145       else if (unformat (input, "user hash buckets %d", &user_buckets))
3146         ;
3147       else if (unformat (input, "user hash memory %d", &user_memory_size))
3148         ;
3149       else if (unformat (input, "max translations per user %d",
3150                          &max_translations_per_user))
3151         ;
3152       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3153         ;
3154       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3155         ;
3156       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3157         ;
3158       else if (unformat (input, "static mapping only"))
3159         {
3160           static_mapping_only = 1;
3161           if (unformat (input, "connection tracking"))
3162             static_mapping_connection_tracking = 1;
3163         }
3164       else if (unformat (input, "deterministic"))
3165         sm->deterministic = 1;
3166       else if (unformat (input, "nat64 bib hash buckets %d",
3167                          &nat64_bib_buckets))
3168         ;
3169       else if (unformat (input, "nat64 bib hash memory %d",
3170                          &nat64_bib_memory_size))
3171         ;
3172       else
3173         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3174         ;
3175       else if (unformat (input, "nat64 st hash memory %d",
3176                          &nat64_st_memory_size))
3177         ;
3178       else if (unformat (input, "out2in dpo"))
3179         sm->out2in_dpo = 1;
3180       else if (unformat (input, "dslite ce"))
3181         dslite_set_ce (dm, 1);
3182       else if (unformat (input, "endpoint-dependent"))
3183         sm->endpoint_dependent = 1;
3184       else
3185         return clib_error_return (0, "unknown input '%U'",
3186                                   format_unformat_error, input);
3187     }
3188
3189   if (sm->deterministic && sm->endpoint_dependent)
3190     return clib_error_return (0,
3191                               "deterministic and endpoint-dependent modes are mutually exclusive");
3192
3193   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3194     return clib_error_return (0,
3195                               "static mapping only mode available only for simple nat");
3196
3197   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3198     return clib_error_return (0,
3199                               "out2in dpo mode available only for simple nat");
3200
3201   /* for show commands, etc. */
3202   sm->translation_buckets = translation_buckets;
3203   sm->translation_memory_size = translation_memory_size;
3204   /* do not exceed load factor 10 */
3205   sm->max_translations = 10 * translation_buckets;
3206   sm->user_buckets = user_buckets;
3207   sm->user_memory_size = user_memory_size;
3208   sm->max_translations_per_user = max_translations_per_user;
3209   sm->outside_vrf_id = outside_vrf_id;
3210   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3211                                                              outside_vrf_id,
3212                                                              FIB_SOURCE_PLUGIN_HI);
3213   nm->outside_vrf_id = outside_ip6_vrf_id;
3214   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3215                                                              outside_ip6_vrf_id,
3216                                                              FIB_SOURCE_PLUGIN_HI);
3217   sm->inside_vrf_id = inside_vrf_id;
3218   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3219                                                             inside_vrf_id,
3220                                                             FIB_SOURCE_PLUGIN_HI);
3221   sm->static_mapping_only = static_mapping_only;
3222   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3223
3224   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3225                   nat64_st_memory_size);
3226
3227   if (sm->deterministic)
3228     {
3229       sm->in2out_node_index = snat_det_in2out_node.index;
3230       sm->in2out_output_node_index = ~0;
3231       sm->out2in_node_index = snat_det_out2in_node.index;
3232       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3233       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3234     }
3235   else
3236     {
3237       if (sm->endpoint_dependent)
3238         {
3239           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3240           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3241           sm->in2out_node_index = nat44_ed_in2out_node.index;
3242           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3243           sm->out2in_node_index = nat44_ed_out2in_node.index;
3244           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3245           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3246           nat_affinity_init (vm);
3247         }
3248       else
3249         {
3250           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3251           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3252           sm->in2out_node_index = snat_in2out_node.index;
3253           sm->in2out_output_node_index = snat_in2out_output_node.index;
3254           sm->out2in_node_index = snat_out2in_node.index;
3255           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3256           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3257         }
3258       if (!static_mapping_only ||
3259           (static_mapping_only && static_mapping_connection_tracking))
3260         {
3261           /* *INDENT-OFF* */
3262           vec_foreach (tsm, sm->per_thread_data)
3263             {
3264               if (sm->endpoint_dependent)
3265                 {
3266                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3267                                          translation_buckets,
3268                                          translation_memory_size);
3269                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3270                                                       format_ed_session_kvp);
3271
3272                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3273                                          translation_buckets,
3274                                          translation_memory_size);
3275                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3276                                                       format_ed_session_kvp);
3277                 }
3278               else
3279                 {
3280                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3281                                         translation_buckets,
3282                                         translation_memory_size);
3283                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3284                                                      format_session_kvp);
3285
3286                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3287                                         translation_buckets,
3288                                         translation_memory_size);
3289                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3290                                                      format_session_kvp);
3291                 }
3292
3293               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3294                                     user_memory_size);
3295               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3296                                                  format_user_kvp);
3297             }
3298           /* *INDENT-ON* */
3299
3300         }
3301       else
3302         {
3303           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3304           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3305         }
3306       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3307                             "static_mapping_by_local", static_mapping_buckets,
3308                             static_mapping_memory_size);
3309       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3310                                          format_static_mapping_kvp);
3311
3312       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3313                             "static_mapping_by_external",
3314                             static_mapping_buckets,
3315                             static_mapping_memory_size);
3316       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3317                                          format_static_mapping_kvp);
3318     }
3319
3320   return 0;
3321 }
3322
3323 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3324
3325 static void
3326 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3327                                  uword opaque,
3328                                  u32 sw_if_index,
3329                                  ip4_address_t * address,
3330                                  u32 address_length,
3331                                  u32 if_address_index, u32 is_delete)
3332 {
3333   snat_main_t *sm = &snat_main;
3334   snat_static_map_resolve_t *rp;
3335   snat_static_mapping_t *m;
3336   snat_session_key_t m_key;
3337   clib_bihash_kv_8_8_t kv, value;
3338   int i, rv;
3339   ip4_address_t l_addr;
3340
3341   for (i = 0; i < vec_len (sm->to_resolve); i++)
3342     {
3343       rp = sm->to_resolve + i;
3344       if (rp->addr_only == 0)
3345         continue;
3346       if (rp->sw_if_index == sw_if_index)
3347         goto match;
3348     }
3349
3350   return;
3351
3352 match:
3353   m_key.addr.as_u32 = address->as_u32;
3354   m_key.port = rp->addr_only ? 0 : rp->e_port;
3355   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3356   m_key.fib_index = sm->outside_fib_index;
3357   kv.key = m_key.as_u64;
3358   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3359     m = 0;
3360   else
3361     m = pool_elt_at_index (sm->static_mappings, value.value);
3362
3363   if (!is_delete)
3364     {
3365       /* Don't trip over lease renewal, static config */
3366       if (m)
3367         return;
3368     }
3369   else
3370     {
3371       if (!m)
3372         return;
3373     }
3374
3375   /* Indetity mapping? */
3376   if (rp->l_addr.as_u32 == 0)
3377     l_addr.as_u32 = address[0].as_u32;
3378   else
3379     l_addr.as_u32 = rp->l_addr.as_u32;
3380   /* Add the static mapping */
3381   rv = snat_add_static_mapping (l_addr,
3382                                 address[0],
3383                                 rp->l_port,
3384                                 rp->e_port,
3385                                 rp->vrf_id,
3386                                 rp->addr_only, ~0 /* sw_if_index */ ,
3387                                 rp->proto, !is_delete, rp->twice_nat,
3388                                 rp->out2in_only, rp->tag, rp->identity_nat);
3389   if (rv)
3390     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3391 }
3392
3393 static void
3394 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3395                                        uword opaque,
3396                                        u32 sw_if_index,
3397                                        ip4_address_t * address,
3398                                        u32 address_length,
3399                                        u32 if_address_index, u32 is_delete)
3400 {
3401   snat_main_t *sm = &snat_main;
3402   snat_static_map_resolve_t *rp;
3403   ip4_address_t l_addr;
3404   int i, j;
3405   int rv;
3406   u8 twice_nat = 0;
3407   snat_address_t *addresses = sm->addresses;
3408
3409   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3410     {
3411       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3412         goto match;
3413     }
3414
3415   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3416     {
3417       twice_nat = 1;
3418       addresses = sm->twice_nat_addresses;
3419       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3420         goto match;
3421     }
3422
3423   return;
3424
3425 match:
3426   if (!is_delete)
3427     {
3428       /* Don't trip over lease renewal, static config */
3429       for (j = 0; j < vec_len (addresses); j++)
3430         if (addresses[j].addr.as_u32 == address->as_u32)
3431           return;
3432
3433       (void) snat_add_address (sm, address, ~0, twice_nat);
3434       /* Scan static map resolution vector */
3435       for (j = 0; j < vec_len (sm->to_resolve); j++)
3436         {
3437           rp = sm->to_resolve + j;
3438           if (rp->addr_only)
3439             continue;
3440           /* On this interface? */
3441           if (rp->sw_if_index == sw_if_index)
3442             {
3443               /* Indetity mapping? */
3444               if (rp->l_addr.as_u32 == 0)
3445                 l_addr.as_u32 = address[0].as_u32;
3446               else
3447                 l_addr.as_u32 = rp->l_addr.as_u32;
3448               /* Add the static mapping */
3449               rv = snat_add_static_mapping (l_addr,
3450                                             address[0],
3451                                             rp->l_port,
3452                                             rp->e_port,
3453                                             rp->vrf_id,
3454                                             rp->addr_only,
3455                                             ~0 /* sw_if_index */ ,
3456                                             rp->proto,
3457                                             rp->is_add, rp->twice_nat,
3458                                             rp->out2in_only, rp->tag,
3459                                             rp->identity_nat);
3460               if (rv)
3461                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3462             }
3463         }
3464       return;
3465     }
3466   else
3467     {
3468       (void) snat_del_address (sm, address[0], 1, twice_nat);
3469       return;
3470     }
3471 }
3472
3473
3474 int
3475 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3476                             u8 twice_nat)
3477 {
3478   ip4_main_t *ip4_main = sm->ip4_main;
3479   ip4_address_t *first_int_addr;
3480   snat_static_map_resolve_t *rp;
3481   u32 *indices_to_delete = 0;
3482   int i, j;
3483   u32 *auto_add_sw_if_indices =
3484     twice_nat ? sm->
3485     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3486
3487   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3488     );
3489
3490   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3491     {
3492       if (auto_add_sw_if_indices[i] == sw_if_index)
3493         {
3494           if (is_del)
3495             {
3496               /* if have address remove it */
3497               if (first_int_addr)
3498                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3499               else
3500                 {
3501                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3502                     {
3503                       rp = sm->to_resolve + j;
3504                       if (rp->sw_if_index == sw_if_index)
3505                         vec_add1 (indices_to_delete, j);
3506                     }
3507                   if (vec_len (indices_to_delete))
3508                     {
3509                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3510                         vec_del1 (sm->to_resolve, j);
3511                       vec_free (indices_to_delete);
3512                     }
3513                 }
3514               if (twice_nat)
3515                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3516               else
3517                 vec_del1 (sm->auto_add_sw_if_indices, i);
3518             }
3519           else
3520             return VNET_API_ERROR_VALUE_EXIST;
3521
3522           return 0;
3523         }
3524     }
3525
3526   if (is_del)
3527     return VNET_API_ERROR_NO_SUCH_ENTRY;
3528
3529   /* add to the auto-address list */
3530   if (twice_nat)
3531     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3532   else
3533     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3534
3535   /* If the address is already bound - or static - add it now */
3536   if (first_int_addr)
3537     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3538
3539   return 0;
3540 }
3541
3542 int
3543 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3544                    snat_protocol_t proto, u32 vrf_id, int is_in)
3545 {
3546   snat_main_per_thread_data_t *tsm;
3547   clib_bihash_kv_8_8_t kv, value;
3548   ip4_header_t ip;
3549   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3550   snat_session_key_t key;
3551   snat_session_t *s;
3552   clib_bihash_8_8_t *t;
3553
3554   if (sm->endpoint_dependent)
3555     return VNET_API_ERROR_UNSUPPORTED;
3556
3557   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3558   if (sm->num_workers > 1)
3559     tsm =
3560       vec_elt_at_index (sm->per_thread_data,
3561                         sm->worker_in2out_cb (&ip, fib_index));
3562   else
3563     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3564
3565   key.addr.as_u32 = addr->as_u32;
3566   key.port = clib_host_to_net_u16 (port);
3567   key.protocol = proto;
3568   key.fib_index = fib_index;
3569   kv.key = key.as_u64;
3570   t = is_in ? &tsm->in2out : &tsm->out2in;
3571   if (!clib_bihash_search_8_8 (t, &kv, &value))
3572     {
3573       if (pool_is_free_index (tsm->sessions, value.value))
3574         return VNET_API_ERROR_UNSPECIFIED;
3575
3576       s = pool_elt_at_index (tsm->sessions, value.value);
3577       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3578       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3579       return 0;
3580     }
3581
3582   return VNET_API_ERROR_NO_SUCH_ENTRY;
3583 }
3584
3585 int
3586 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3587                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3588                       u32 vrf_id, int is_in)
3589 {
3590   ip4_header_t ip;
3591   clib_bihash_16_8_t *t;
3592   nat_ed_ses_key_t key;
3593   clib_bihash_kv_16_8_t kv, value;
3594   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3595   snat_session_t *s;
3596   snat_main_per_thread_data_t *tsm;
3597
3598   if (!sm->endpoint_dependent)
3599     return VNET_API_ERROR_FEATURE_DISABLED;
3600
3601   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3602   if (sm->num_workers > 1)
3603     tsm =
3604       vec_elt_at_index (sm->per_thread_data,
3605                         sm->worker_in2out_cb (&ip, fib_index));
3606   else
3607     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3608
3609   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3610   key.l_addr.as_u32 = addr->as_u32;
3611   key.r_addr.as_u32 = eh_addr->as_u32;
3612   key.l_port = clib_host_to_net_u16 (port);
3613   key.r_port = clib_host_to_net_u16 (eh_port);
3614   key.proto = proto;
3615   key.fib_index = fib_index;
3616   kv.key[0] = key.as_u64[0];
3617   kv.key[1] = key.as_u64[1];
3618   if (clib_bihash_search_16_8 (t, &kv, &value))
3619     return VNET_API_ERROR_NO_SUCH_ENTRY;
3620
3621   if (pool_is_free_index (tsm->sessions, value.value))
3622     return VNET_API_ERROR_UNSPECIFIED;
3623   s = pool_elt_at_index (tsm->sessions, value.value);
3624   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3625   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3626   return 0;
3627 }
3628
3629 void
3630 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3631 {
3632   snat_main_t *sm = &snat_main;
3633
3634   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3635   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3636   sm->psid = psid;
3637   sm->psid_offset = psid_offset;
3638   sm->psid_length = psid_length;
3639 }
3640
3641 void
3642 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3643 {
3644   snat_main_t *sm = &snat_main;
3645
3646   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3647   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3648   sm->start_port = start_port;
3649   sm->end_port = end_port;
3650 }
3651
3652 void
3653 nat_set_alloc_addr_and_port_default (void)
3654 {
3655   snat_main_t *sm = &snat_main;
3656
3657   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3658   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3659 }
3660
3661 /*
3662  * fd.io coding-style-patch-verification: ON
3663  *
3664  * Local Variables:
3665  * eval: (c-set-style "gnu")
3666  * End:
3667  */