src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35
  36 #include <vpp/app/version.h>
  37
  38 snat_main_t snat_main;
  39
  40 /* *INDENT-OFF* */
  41
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat44-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  47 };
  48 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat44-out2in",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-dhcp-client-detect"),
  53 };
  54 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-classify",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-in2out",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-out2in",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-det-classify",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  74 };
  75 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-ed-in2out",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-ed-out2in",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  84                                "ip4-dhcp-client-detect"),
  85 };
  86 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-ed-classify",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-in2out-worker-handoff",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  95 };
  96 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-out2in-worker-handoff",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-handoff-classify",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-in2out-fast",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 111 };
 112 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-out2in-fast",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 116                                "ip4-dhcp-client-detect"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-hairpin-dst",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ed-hairpin-dst",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 127 };
 128
 129 /* Hook up output features */
 130 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 131   .arc_name = "ip4-output",
 132   .node_name = "nat44-in2out-output",
 133   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 136   .arc_name = "ip4-output",
 137   .node_name = "nat44-in2out-output-worker-handoff",
 138   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 141   .arc_name = "ip4-output",
 142   .node_name = "nat44-hairpin-src",
 143   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 144 };
 145 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-ed-in2out-output",
 148   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 149 };
 150 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-ed-hairpin-src",
 153   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 154 };
 155
 156 /* Hook up ip4-local features */
 157 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 158 {
 159   .arc_name = "ip4-local",
 160   .node_name = "nat44-hairpinning",
 161   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 164 {
 165   .arc_name = "ip4-local",
 166   .node_name = "nat44-ed-hairpinning",
 167   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 168 };
 169
 170
 171 VLIB_PLUGIN_REGISTER () = {
 172     .version = VPP_BUILD_VER,
 173     .description = "Network Address Translation",
 174 };
 175 /* *INDENT-ON* */
 176
 177 void
 178 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 179 {
 180   snat_session_key_t key;
 181   clib_bihash_kv_8_8_t kv;
 182   nat_ed_ses_key_t ed_key;
 183   clib_bihash_kv_16_8_t ed_kv;
 184   snat_main_per_thread_data_t *tsm =
 185     vec_elt_at_index (sm->per_thread_data, thread_index);
 186
 187   if (is_fwd_bypass_session (s))
 188     {
 189       ed_key.l_addr = s->in2out.addr;
 190       ed_key.r_addr = s->ext_host_addr;
 191       ed_key.l_port = s->in2out.port;
 192       ed_key.r_port = s->ext_host_port;
 193       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 194       ed_key.fib_index = 0;
 195       ed_kv.key[0] = ed_key.as_u64[0];
 196       ed_kv.key[1] = ed_key.as_u64[1];
 197       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 198         nat_log_warn ("in2out_ed key del failed");
 199       return;
 200     }
 201
 202   /* session lookup tables */
 203   if (is_ed_session (s))
 204     {
 205       if (is_affinity_sessions (s))
 206         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 207                              s->in2out.protocol, s->out2in.port);
 208       ed_key.l_addr = s->out2in.addr;
 209       ed_key.r_addr = s->ext_host_addr;
 210       ed_key.fib_index = s->out2in.fib_index;
 211       if (snat_is_unk_proto_session (s))
 212         {
 213           ed_key.proto = s->in2out.port;
 214           ed_key.r_port = 0;
 215           ed_key.l_port = 0;
 216         }
 217       else
 218         {
 219           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 220           ed_key.l_port = s->out2in.port;
 221           ed_key.r_port = s->ext_host_port;
 222         }
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 226         nat_log_warn ("out2in_ed key del failed");
 227       ed_key.l_addr = s->in2out.addr;
 228       ed_key.fib_index = s->in2out.fib_index;
 229       if (!snat_is_unk_proto_session (s))
 230         ed_key.l_port = s->in2out.port;
 231       if (is_twice_nat_session (s))
 232         {
 233           ed_key.r_addr = s->ext_host_nat_addr;
 234           ed_key.r_port = s->ext_host_nat_port;
 235         }
 236       ed_kv.key[0] = ed_key.as_u64[0];
 237       ed_kv.key[1] = ed_key.as_u64[1];
 238       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 239         nat_log_warn ("in2out_ed key del failed");
 240
 241       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 242                              &s->in2out.addr, s->in2out.port,
 243                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 244                              &s->out2in.addr, s->out2in.port,
 245                              &s->ext_host_addr, s->ext_host_port,
 246                              s->in2out.protocol, is_twice_nat_session (s));
 247     }
 248   else
 249     {
 250       kv.key = s->in2out.as_u64;
 251       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 252         nat_log_warn ("in2out key del failed");
 253       kv.key = s->out2in.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 255         nat_log_warn ("out2in key del failed");
 256
 257       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 258                                &s->in2out.addr, s->in2out.port,
 259                                &s->out2in.addr, s->out2in.port,
 260                                s->in2out.protocol);
 261     }
 262
 263   if (snat_is_unk_proto_session (s))
 264     return;
 265
 266   /* log NAT event */
 267   snat_ipfix_logging_nat44_ses_delete (thread_index,
 268                                        s->in2out.addr.as_u32,
 269                                        s->out2in.addr.as_u32,
 270                                        s->in2out.protocol,
 271                                        s->in2out.port,
 272                                        s->out2in.port, s->in2out.fib_index);
 273
 274   /* Twice NAT address and port for external host */
 275   if (is_twice_nat_session (s))
 276     {
 277       key.protocol = s->in2out.protocol;
 278       key.port = s->ext_host_nat_port;
 279       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 280       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 281                                           thread_index, &key);
 282     }
 283
 284   if (snat_is_session_static (s))
 285     return;
 286
 287   snat_free_outside_address_and_port (sm->addresses, thread_index,
 288                                       &s->out2in);
 289 }
 290
 291 snat_user_t *
 292 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 293                         u32 thread_index)
 294 {
 295   snat_user_t *u = 0;
 296   snat_user_key_t user_key;
 297   clib_bihash_kv_8_8_t kv, value;
 298   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 299   dlist_elt_t *per_user_list_head_elt;
 300
 301   user_key.addr.as_u32 = addr->as_u32;
 302   user_key.fib_index = fib_index;
 303   kv.key = user_key.as_u64;
 304
 305   /* Ever heard of the "user" = src ip4 address before? */
 306   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 307     {
 308       /* no, make a new one */
 309       pool_get (tsm->users, u);
 310       clib_memset (u, 0, sizeof (*u));
 311       u->addr.as_u32 = addr->as_u32;
 312       u->fib_index = fib_index;
 313
 314       pool_get (tsm->list_pool, per_user_list_head_elt);
 315
 316       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 317         tsm->list_pool;
 318
 319       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 320
 321       kv.value = u - tsm->users;
 322
 323       /* add user */
 324       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 325         nat_log_warn ("user_hash keay add failed");
 326
 327       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 328                                pool_elts (tsm->users));
 329     }
 330   else
 331     {
 332       u = pool_elt_at_index (tsm->users, value.value);
 333     }
 334
 335   return u;
 336 }
 337
 338 snat_session_t *
 339 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 340                               u32 thread_index)
 341 {
 342   snat_session_t *s;
 343   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 344   u32 oldest_per_user_translation_list_index, session_index;
 345   dlist_elt_t *oldest_per_user_translation_list_elt;
 346   dlist_elt_t *per_user_translation_list_elt;
 347
 348   /* Over quota? Recycle the least recently used translation */
 349   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 350     {
 351       oldest_per_user_translation_list_index =
 352         clib_dlist_remove_head (tsm->list_pool,
 353                                 u->sessions_per_user_list_head_index);
 354
 355       ASSERT (oldest_per_user_translation_list_index != ~0);
 356
 357       /* Add it back to the end of the LRU list */
 358       clib_dlist_addtail (tsm->list_pool,
 359                           u->sessions_per_user_list_head_index,
 360                           oldest_per_user_translation_list_index);
 361       /* Get the list element */
 362       oldest_per_user_translation_list_elt =
 363         pool_elt_at_index (tsm->list_pool,
 364                            oldest_per_user_translation_list_index);
 365
 366       /* Get the session index from the list element */
 367       session_index = oldest_per_user_translation_list_elt->value;
 368
 369       /* Get the session */
 370       s = pool_elt_at_index (tsm->sessions, session_index);
 371       nat_free_session_data (sm, s, thread_index);
 372       if (snat_is_session_static (s))
 373         u->nstaticsessions--;
 374       else
 375         u->nsessions--;
 376       s->flags = 0;
 377       s->total_bytes = 0;
 378       s->total_pkts = 0;
 379       s->state = 0;
 380       s->ext_host_addr.as_u32 = 0;
 381       s->ext_host_port = 0;
 382       s->ext_host_nat_addr.as_u32 = 0;
 383       s->ext_host_nat_port = 0;
 384     }
 385   else
 386     {
 387       pool_get (tsm->sessions, s);
 388       clib_memset (s, 0, sizeof (*s));
 389
 390       /* Create list elts */
 391       pool_get (tsm->list_pool, per_user_translation_list_elt);
 392       clib_dlist_init (tsm->list_pool,
 393                        per_user_translation_list_elt - tsm->list_pool);
 394
 395       per_user_translation_list_elt->value = s - tsm->sessions;
 396       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 397       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 398
 399       clib_dlist_addtail (tsm->list_pool,
 400                           s->per_user_list_head_index,
 401                           per_user_translation_list_elt - tsm->list_pool);
 402
 403       s->user_index = u - tsm->users;
 404       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 405                                pool_elts (tsm->sessions));
 406     }
 407
 408   return s;
 409 }
 410
 411 snat_session_t *
 412 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 413                       f64 now)
 414 {
 415   snat_session_t *s;
 416   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 417   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 418   u32 oldest_index;
 419   u64 sess_timeout_time;
 420
 421   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 422     goto alloc_new;
 423
 424   oldest_index =
 425     clib_dlist_remove_head (tsm->list_pool,
 426                             u->sessions_per_user_list_head_index);
 427   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 428   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 429   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 430   if (now >= sess_timeout_time)
 431     {
 432       clib_dlist_addtail (tsm->list_pool,
 433                           u->sessions_per_user_list_head_index, oldest_index);
 434       nat_free_session_data (sm, s, thread_index);
 435       if (snat_is_session_static (s))
 436         u->nstaticsessions--;
 437       else
 438         u->nsessions--;
 439       s->flags = 0;
 440       s->total_bytes = 0;
 441       s->total_pkts = 0;
 442       s->state = 0;
 443       s->ext_host_addr.as_u32 = 0;
 444       s->ext_host_port = 0;
 445       s->ext_host_nat_addr.as_u32 = 0;
 446       s->ext_host_nat_port = 0;
 447     }
 448   else
 449     {
 450       clib_dlist_addhead (tsm->list_pool,
 451                           u->sessions_per_user_list_head_index, oldest_index);
 452       if ((u->nsessions + u->nstaticsessions) >=
 453           sm->max_translations_per_user)
 454         {
 455           nat_log_warn ("max translations per user %U", format_ip4_address,
 456                         &u->addr);
 457           snat_ipfix_logging_max_entries_per_user
 458             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 459           return 0;
 460         }
 461       else
 462         {
 463         alloc_new:
 464           pool_get (tsm->sessions, s);
 465           clib_memset (s, 0, sizeof (*s));
 466
 467           /* Create list elts */
 468           pool_get (tsm->list_pool, per_user_translation_list_elt);
 469           clib_dlist_init (tsm->list_pool,
 470                            per_user_translation_list_elt - tsm->list_pool);
 471
 472           per_user_translation_list_elt->value = s - tsm->sessions;
 473           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 474           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 475
 476           clib_dlist_addtail (tsm->list_pool,
 477                               s->per_user_list_head_index,
 478                               per_user_translation_list_elt - tsm->list_pool);
 479         }
 480
 481       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 482                                pool_elts (tsm->sessions));
 483     }
 484
 485   return s;
 486 }
 487
 488 void
 489 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 490                           int is_add)
 491 {
 492   fib_prefix_t prefix = {
 493     .fp_len = p_len,
 494     .fp_proto = FIB_PROTOCOL_IP4,
 495     .fp_addr = {
 496                 .ip4.as_u32 = addr->as_u32,
 497                 },
 498   };
 499   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 500
 501   if (is_add)
 502     fib_table_entry_update_one_path (fib_index,
 503                                      &prefix,
 504                                      FIB_SOURCE_PLUGIN_LOW,
 505                                      (FIB_ENTRY_FLAG_CONNECTED |
 506                                       FIB_ENTRY_FLAG_LOCAL |
 507                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 508                                      DPO_PROTO_IP4,
 509                                      NULL,
 510                                      sw_if_index,
 511                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 512   else
 513     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 514 }
 515
 516 int
 517 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 518                   u8 twice_nat)
 519 {
 520   snat_address_t *ap;
 521   snat_interface_t *i;
 522   vlib_thread_main_t *tm = vlib_get_thread_main ();
 523
 524   if (twice_nat && !sm->endpoint_dependent)
 525     return VNET_API_ERROR_FEATURE_DISABLED;
 526
 527   /* Check if address already exists */
 528   /* *INDENT-OFF* */
 529   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 530     {
 531       if (ap->addr.as_u32 == addr->as_u32)
 532         return VNET_API_ERROR_VALUE_EXIST;
 533     }
 534   /* *INDENT-ON* */
 535
 536   if (twice_nat)
 537     vec_add2 (sm->twice_nat_addresses, ap, 1);
 538   else
 539     vec_add2 (sm->addresses, ap, 1);
 540
 541   ap->addr = *addr;
 542   if (vrf_id != ~0)
 543     ap->fib_index =
 544       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 545                                          FIB_SOURCE_PLUGIN_LOW);
 546   else
 547     ap->fib_index = ~0;
 548 #define _(N, i, n, s) \
 549   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 550   ap->busy_##n##_ports = 0; \
 551   ap->busy_##n##_ports_per_thread = 0;\
 552   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 553   foreach_snat_protocol
 554 #undef _
 555     if (twice_nat)
 556     return 0;
 557
 558   /* Add external address to FIB */
 559   /* *INDENT-OFF* */
 560   pool_foreach (i, sm->interfaces,
 561   ({
 562     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 563       continue;
 564
 565     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 566     break;
 567   }));
 568   pool_foreach (i, sm->output_feature_interfaces,
 569   ({
 570     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 571       continue;
 572
 573     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 574     break;
 575   }));
 576   /* *INDENT-ON* */
 577
 578   return 0;
 579 }
 580
 581 static int
 582 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 583 {
 584   snat_static_mapping_t *m;
 585   /* *INDENT-OFF* */
 586   pool_foreach (m, sm->static_mappings,
 587   ({
 588       if (is_addr_only_static_mapping (m) ||
 589           is_out2in_only_static_mapping (m) ||
 590           is_identity_static_mapping (m))
 591         continue;
 592       if (m->external_addr.as_u32 == addr.as_u32)
 593         return 1;
 594   }));
 595   /* *INDENT-ON* */
 596
 597   return 0;
 598 }
 599
 600 void
 601 increment_v4_address (ip4_address_t * a)
 602 {
 603   u32 v;
 604
 605   v = clib_net_to_host_u32 (a->as_u32) + 1;
 606   a->as_u32 = clib_host_to_net_u32 (v);
 607 }
 608
 609 static void
 610 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 611                                        ip4_address_t l_addr,
 612                                        u16 l_port,
 613                                        u32 sw_if_index,
 614                                        u16 e_port,
 615                                        u32 vrf_id,
 616                                        snat_protocol_t proto,
 617                                        int addr_only, int is_add, u8 * tag,
 618                                        int twice_nat, int out2in_only,
 619                                        int identity_nat)
 620 {
 621   snat_static_map_resolve_t *rp;
 622
 623   vec_add2 (sm->to_resolve, rp, 1);
 624   rp->l_addr.as_u32 = l_addr.as_u32;
 625   rp->l_port = l_port;
 626   rp->sw_if_index = sw_if_index;
 627   rp->e_port = e_port;
 628   rp->vrf_id = vrf_id;
 629   rp->proto = proto;
 630   rp->addr_only = addr_only;
 631   rp->is_add = is_add;
 632   rp->twice_nat = twice_nat;
 633   rp->out2in_only = out2in_only;
 634   rp->identity_nat = identity_nat;
 635   rp->tag = vec_dup (tag);
 636 }
 637
 638 static u32
 639 get_thread_idx_by_port (u16 e_port)
 640 {
 641   snat_main_t *sm = &snat_main;
 642   u32 thread_idx = sm->num_workers;
 643   if (sm->num_workers > 1)
 644     {
 645       thread_idx =
 646         sm->first_worker_index +
 647         sm->workers[(e_port - 1024) / sm->port_per_thread];
 648     }
 649   return thread_idx;
 650 }
 651
 652 int
 653 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 654                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 655                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 656                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 657                          u8 identity_nat)
 658 {
 659   snat_main_t *sm = &snat_main;
 660   snat_static_mapping_t *m;
 661   snat_session_key_t m_key;
 662   clib_bihash_kv_8_8_t kv, value;
 663   snat_address_t *a = 0;
 664   u32 fib_index = ~0;
 665   snat_interface_t *interface;
 666   int i;
 667   snat_main_per_thread_data_t *tsm;
 668   snat_user_key_t u_key;
 669   snat_user_t *u;
 670   dlist_elt_t *head, *elt;
 671   u32 elt_index, head_index;
 672   u32 ses_index;
 673   u64 user_index;
 674   snat_session_t *s;
 675   snat_static_map_resolve_t *rp, *rp_match = 0;
 676   nat44_lb_addr_port_t *local;
 677   u32 find = ~0;
 678
 679   if (!sm->endpoint_dependent)
 680     {
 681       if (twice_nat || out2in_only)
 682         return VNET_API_ERROR_FEATURE_DISABLED;
 683     }
 684
 685   /* If the external address is a specific interface address */
 686   if (sw_if_index != ~0)
 687     {
 688       ip4_address_t *first_int_addr;
 689
 690       for (i = 0; i < vec_len (sm->to_resolve); i++)
 691         {
 692           rp = sm->to_resolve + i;
 693           if (rp->sw_if_index != sw_if_index ||
 694               rp->l_addr.as_u32 != l_addr.as_u32 ||
 695               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 696             continue;
 697
 698           if (!addr_only)
 699             {
 700               if (rp->l_port != l_port || rp->e_port != e_port
 701                   || rp->proto != proto)
 702                 continue;
 703             }
 704
 705           rp_match = rp;
 706           break;
 707         }
 708
 709       /* Might be already set... */
 710       first_int_addr = ip4_interface_first_address
 711         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 712
 713       if (is_add)
 714         {
 715           if (rp_match)
 716             return VNET_API_ERROR_VALUE_EXIST;
 717
 718           snat_add_static_mapping_when_resolved
 719             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 720              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 721
 722           /* DHCP resolution required? */
 723           if (first_int_addr == 0)
 724             {
 725               return 0;
 726             }
 727           else
 728             {
 729               e_addr.as_u32 = first_int_addr->as_u32;
 730               /* Identity mapping? */
 731               if (l_addr.as_u32 == 0)
 732                 l_addr.as_u32 = e_addr.as_u32;
 733             }
 734         }
 735       else
 736         {
 737           if (!rp_match)
 738             return VNET_API_ERROR_NO_SUCH_ENTRY;
 739
 740           vec_del1 (sm->to_resolve, i);
 741
 742           if (first_int_addr)
 743             {
 744               e_addr.as_u32 = first_int_addr->as_u32;
 745               /* Identity mapping? */
 746               if (l_addr.as_u32 == 0)
 747                 l_addr.as_u32 = e_addr.as_u32;
 748             }
 749           else
 750             return 0;
 751         }
 752     }
 753
 754   m_key.addr = e_addr;
 755   m_key.port = addr_only ? 0 : e_port;
 756   m_key.protocol = addr_only ? 0 : proto;
 757   m_key.fib_index = 0;
 758   kv.key = m_key.as_u64;
 759   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 760     m = 0;
 761   else
 762     m = pool_elt_at_index (sm->static_mappings, value.value);
 763
 764   if (is_add)
 765     {
 766       if (m)
 767         {
 768           if (is_identity_static_mapping (m))
 769             {
 770               /* *INDENT-OFF* */
 771               pool_foreach (local, m->locals,
 772               ({
 773                 if (local->vrf_id == vrf_id)
 774                   return VNET_API_ERROR_VALUE_EXIST;
 775               }));
 776               /* *INDENT-ON* */
 777               pool_get (m->locals, local);
 778               local->vrf_id = vrf_id;
 779               local->fib_index =
 780                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 781                                                    FIB_SOURCE_PLUGIN_LOW);
 782               m_key.addr = m->local_addr;
 783               m_key.port = m->local_port;
 784               m_key.protocol = m->proto;
 785               m_key.fib_index = local->fib_index;
 786               kv.key = m_key.as_u64;
 787               kv.value = m - sm->static_mappings;
 788               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 789               return 0;
 790             }
 791           else
 792             return VNET_API_ERROR_VALUE_EXIST;
 793         }
 794
 795       if (twice_nat && addr_only)
 796         return VNET_API_ERROR_UNSUPPORTED;
 797
 798       /* Convert VRF id to FIB index */
 799       if (vrf_id != ~0)
 800         fib_index =
 801           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 802                                              FIB_SOURCE_PLUGIN_LOW);
 803       /* If not specified use inside VRF id from SNAT plugin startup config */
 804       else
 805         {
 806           fib_index = sm->inside_fib_index;
 807           vrf_id = sm->inside_vrf_id;
 808         }
 809
 810       if (!(out2in_only || identity_nat))
 811         {
 812           m_key.addr = l_addr;
 813           m_key.port = addr_only ? 0 : l_port;
 814           m_key.protocol = addr_only ? 0 : proto;
 815           m_key.fib_index = fib_index;
 816           kv.key = m_key.as_u64;
 817           if (!clib_bihash_search_8_8
 818               (&sm->static_mapping_by_local, &kv, &value))
 819             return VNET_API_ERROR_VALUE_EXIST;
 820         }
 821
 822       /* Find external address in allocated addresses and reserve port for
 823          address and port pair mapping when dynamic translations enabled */
 824       if (!(addr_only || sm->static_mapping_only || out2in_only))
 825         {
 826           for (i = 0; i < vec_len (sm->addresses); i++)
 827             {
 828               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 829                 {
 830                   a = sm->addresses + i;
 831                   /* External port must be unused */
 832                   switch (proto)
 833                     {
 834 #define _(N, j, n, s) \
 835                     case SNAT_PROTOCOL_##N: \
 836                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 837                         return VNET_API_ERROR_INVALID_VALUE; \
 838                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 839                       if (e_port > 1024) \
 840                         { \
 841                           a->busy_##n##_ports++; \
 842                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 843                         } \
 844                       break;
 845                       foreach_snat_protocol
 846 #undef _
 847                     default:
 848                       nat_log_info ("unknown protocol");
 849                       return VNET_API_ERROR_INVALID_VALUE_2;
 850                     }
 851                   break;
 852                 }
 853             }
 854           /* External address must be allocated */
 855           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 856             {
 857               if (sw_if_index != ~0)
 858                 {
 859                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 860                     {
 861                       rp = sm->to_resolve + i;
 862                       if (rp->addr_only)
 863                         continue;
 864                       if (rp->sw_if_index != sw_if_index &&
 865                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 866                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 867                           rp->e_port != e_port && rp->proto != proto)
 868                         continue;
 869
 870                       vec_del1 (sm->to_resolve, i);
 871                       break;
 872                     }
 873                 }
 874               return VNET_API_ERROR_NO_SUCH_ENTRY;
 875             }
 876         }
 877
 878       pool_get (sm->static_mappings, m);
 879       clib_memset (m, 0, sizeof (*m));
 880       m->tag = vec_dup (tag);
 881       m->local_addr = l_addr;
 882       m->external_addr = e_addr;
 883       m->twice_nat = twice_nat;
 884       if (out2in_only)
 885         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 886       if (addr_only)
 887         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 888       if (identity_nat)
 889         {
 890           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 891           pool_get (m->locals, local);
 892           local->vrf_id = vrf_id;
 893           local->fib_index = fib_index;
 894         }
 895       else
 896         {
 897           m->vrf_id = vrf_id;
 898           m->fib_index = fib_index;
 899         }
 900       if (!addr_only)
 901         {
 902           m->local_port = l_port;
 903           m->external_port = e_port;
 904           m->proto = proto;
 905         }
 906
 907       if (sm->num_workers > 1)
 908         {
 909           ip4_header_t ip = {
 910             .src_address = m->local_addr,
 911           };
 912           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 913           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 914         }
 915       else
 916         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 917
 918       m_key.addr = m->local_addr;
 919       m_key.port = m->local_port;
 920       m_key.protocol = m->proto;
 921       m_key.fib_index = fib_index;
 922       kv.key = m_key.as_u64;
 923       kv.value = m - sm->static_mappings;
 924       if (!out2in_only)
 925         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 926
 927       m_key.addr = m->external_addr;
 928       m_key.port = m->external_port;
 929       m_key.fib_index = 0;
 930       kv.key = m_key.as_u64;
 931       kv.value = m - sm->static_mappings;
 932       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 933
 934       /* Delete dynamic sessions matching local address (+ local port) */
 935       if (!(sm->static_mapping_only))
 936         {
 937           u_key.addr = m->local_addr;
 938           u_key.fib_index = m->fib_index;
 939           kv.key = u_key.as_u64;
 940           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 941             {
 942               user_index = value.value;
 943               u = pool_elt_at_index (tsm->users, user_index);
 944               if (u->nsessions)
 945                 {
 946                   head_index = u->sessions_per_user_list_head_index;
 947                   head = pool_elt_at_index (tsm->list_pool, head_index);
 948                   elt_index = head->next;
 949                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 950                   ses_index = elt->value;
 951                   while (ses_index != ~0)
 952                     {
 953                       s = pool_elt_at_index (tsm->sessions, ses_index);
 954                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 955                       ses_index = elt->value;
 956
 957                       if (snat_is_session_static (s))
 958                         continue;
 959
 960                       if (!addr_only
 961                           && (clib_net_to_host_u16 (s->in2out.port) !=
 962                               m->local_port))
 963                         continue;
 964
 965                       nat_free_session_data (sm, s,
 966                                              tsm - sm->per_thread_data);
 967                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 968
 969                       if (!addr_only && !sm->endpoint_dependent)
 970                         break;
 971                     }
 972                 }
 973             }
 974         }
 975     }
 976   else
 977     {
 978       if (!m)
 979         {
 980           if (sw_if_index != ~0)
 981             return 0;
 982           else
 983             return VNET_API_ERROR_NO_SUCH_ENTRY;
 984         }
 985
 986       if (identity_nat)
 987         {
 988           if (vrf_id == ~0)
 989             vrf_id = sm->inside_vrf_id;
 990
 991           /* *INDENT-OFF* */
 992           pool_foreach (local, m->locals,
 993           ({
 994             if (local->vrf_id == vrf_id)
 995               find = local - m->locals;
 996           }));
 997           /* *INDENT-ON* */
 998           if (find == ~0)
 999             return VNET_API_ERROR_NO_SUCH_ENTRY;
1000
1001           local = pool_elt_at_index (m->locals, find);
1002           fib_index = local->fib_index;
1003           pool_put (m->locals, local);
1004         }
1005       else
1006         fib_index = m->fib_index;
1007
1008       /* Free external address port */
1009       if (!(addr_only || sm->static_mapping_only || out2in_only))
1010         {
1011           for (i = 0; i < vec_len (sm->addresses); i++)
1012             {
1013               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1014                 {
1015                   a = sm->addresses + i;
1016                   switch (proto)
1017                     {
1018 #define _(N, j, n, s) \
1019                     case SNAT_PROTOCOL_##N: \
1020                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1021                       if (e_port > 1024) \
1022                         { \
1023                           a->busy_##n##_ports--; \
1024                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1025                         } \
1026                       break;
1027                       foreach_snat_protocol
1028 #undef _
1029                     default:
1030                       nat_log_info ("unknown protocol");
1031                       return VNET_API_ERROR_INVALID_VALUE_2;
1032                     }
1033                   break;
1034                 }
1035             }
1036         }
1037
1038       if (sm->num_workers > 1)
1039         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1040       else
1041         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1042
1043       m_key.addr = m->local_addr;
1044       m_key.port = m->local_port;
1045       m_key.protocol = m->proto;
1046       m_key.fib_index = fib_index;
1047       kv.key = m_key.as_u64;
1048       if (!out2in_only)
1049         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1050
1051       /* Delete session(s) for static mapping if exist */
1052       if (!(sm->static_mapping_only) ||
1053           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1054         {
1055           u_key.addr = m->local_addr;
1056           u_key.fib_index = fib_index;
1057           kv.key = u_key.as_u64;
1058           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1059             {
1060               user_index = value.value;
1061               u = pool_elt_at_index (tsm->users, user_index);
1062               if (u->nstaticsessions)
1063                 {
1064                   head_index = u->sessions_per_user_list_head_index;
1065                   head = pool_elt_at_index (tsm->list_pool, head_index);
1066                   elt_index = head->next;
1067                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1068                   ses_index = elt->value;
1069                   while (ses_index != ~0)
1070                     {
1071                       s = pool_elt_at_index (tsm->sessions, ses_index);
1072                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1073                       ses_index = elt->value;
1074
1075                       if (!addr_only)
1076                         {
1077                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1078                               (clib_net_to_host_u16 (s->out2in.port) !=
1079                                e_port))
1080                             continue;
1081                         }
1082
1083                       if (is_lb_session (s))
1084                         continue;
1085
1086                       if (!snat_is_session_static (s))
1087                         continue;
1088
1089                       nat_free_session_data (sm, s,
1090                                              tsm - sm->per_thread_data);
1091                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1092
1093                       if (!addr_only && !sm->endpoint_dependent)
1094                         break;
1095                     }
1096                 }
1097             }
1098         }
1099
1100       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1101       if (pool_elts (m->locals))
1102         return 0;
1103
1104       m_key.addr = m->external_addr;
1105       m_key.port = m->external_port;
1106       m_key.fib_index = 0;
1107       kv.key = m_key.as_u64;
1108       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1109
1110       vec_free (m->tag);
1111       vec_free (m->workers);
1112       /* Delete static mapping from pool */
1113       pool_put (sm->static_mappings, m);
1114     }
1115
1116   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1117     return 0;
1118
1119   /* Add/delete external address to FIB */
1120   /* *INDENT-OFF* */
1121   pool_foreach (interface, sm->interfaces,
1122   ({
1123     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1124       continue;
1125
1126     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1127     break;
1128   }));
1129   pool_foreach (interface, sm->output_feature_interfaces,
1130   ({
1131     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1132       continue;
1133
1134     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1135     break;
1136   }));
1137   /* *INDENT-ON* */
1138
1139   return 0;
1140 }
1141
1142 int
1143 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1144                                  snat_protocol_t proto,
1145                                  nat44_lb_addr_port_t * locals, u8 is_add,
1146                                  twice_nat_type_t twice_nat, u8 out2in_only,
1147                                  u8 * tag, u32 affinity)
1148 {
1149   snat_main_t *sm = &snat_main;
1150   snat_static_mapping_t *m;
1151   snat_session_key_t m_key;
1152   clib_bihash_kv_8_8_t kv, value;
1153   snat_address_t *a = 0;
1154   int i;
1155   nat44_lb_addr_port_t *local;
1156   u32 elt_index, head_index, ses_index;
1157   snat_main_per_thread_data_t *tsm;
1158   snat_user_key_t u_key;
1159   snat_user_t *u;
1160   snat_session_t *s;
1161   dlist_elt_t *head, *elt;
1162   uword *bitmap = 0;
1163
1164   if (!sm->endpoint_dependent)
1165     return VNET_API_ERROR_FEATURE_DISABLED;
1166
1167   m_key.addr = e_addr;
1168   m_key.port = e_port;
1169   m_key.protocol = proto;
1170   m_key.fib_index = 0;
1171   kv.key = m_key.as_u64;
1172   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1173     m = 0;
1174   else
1175     m = pool_elt_at_index (sm->static_mappings, value.value);
1176
1177   if (is_add)
1178     {
1179       if (m)
1180         return VNET_API_ERROR_VALUE_EXIST;
1181
1182       if (vec_len (locals) < 2)
1183         return VNET_API_ERROR_INVALID_VALUE;
1184
1185       /* Find external address in allocated addresses and reserve port for
1186          address and port pair mapping when dynamic translations enabled */
1187       if (!(sm->static_mapping_only || out2in_only))
1188         {
1189           for (i = 0; i < vec_len (sm->addresses); i++)
1190             {
1191               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1192                 {
1193                   a = sm->addresses + i;
1194                   /* External port must be unused */
1195                   switch (proto)
1196                     {
1197 #define _(N, j, n, s) \
1198                     case SNAT_PROTOCOL_##N: \
1199                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1200                         return VNET_API_ERROR_INVALID_VALUE; \
1201                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1202                       if (e_port > 1024) \
1203                         { \
1204                           a->busy_##n##_ports++; \
1205                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1206                         } \
1207                       break;
1208                       foreach_snat_protocol
1209 #undef _
1210                     default:
1211                       nat_log_info ("unknown protocol");
1212                       return VNET_API_ERROR_INVALID_VALUE_2;
1213                     }
1214                   break;
1215                 }
1216             }
1217           /* External address must be allocated */
1218           if (!a)
1219             return VNET_API_ERROR_NO_SUCH_ENTRY;
1220         }
1221
1222       pool_get (sm->static_mappings, m);
1223       clib_memset (m, 0, sizeof (*m));
1224       m->tag = vec_dup (tag);
1225       m->external_addr = e_addr;
1226       m->external_port = e_port;
1227       m->proto = proto;
1228       m->twice_nat = twice_nat;
1229       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1230       if (out2in_only)
1231         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1232       m->affinity = affinity;
1233
1234       if (affinity)
1235         m->affinity_per_service_list_head_index =
1236           nat_affinity_get_per_service_list_head_index ();
1237       else
1238         m->affinity_per_service_list_head_index = ~0;
1239
1240       m_key.addr = m->external_addr;
1241       m_key.port = m->external_port;
1242       m_key.protocol = m->proto;
1243       m_key.fib_index = 0;
1244       kv.key = m_key.as_u64;
1245       kv.value = m - sm->static_mappings;
1246       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1247         {
1248           nat_log_err ("static_mapping_by_external key add failed");
1249           return VNET_API_ERROR_UNSPECIFIED;
1250         }
1251
1252       m_key.fib_index = m->fib_index;
1253       for (i = 0; i < vec_len (locals); i++)
1254         {
1255           locals[i].fib_index =
1256             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1257                                                locals[i].vrf_id,
1258                                                FIB_SOURCE_PLUGIN_LOW);
1259           m_key.addr = locals[i].addr;
1260           m_key.fib_index = locals[i].fib_index;
1261           if (!out2in_only)
1262             {
1263               m_key.port = locals[i].port;
1264               kv.key = m_key.as_u64;
1265               kv.value = m - sm->static_mappings;
1266               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1267             }
1268           locals[i].prefix = (i == 0) ? locals[i].probability :
1269             (locals[i - 1].prefix + locals[i].probability);
1270           pool_get (m->locals, local);
1271           *local = locals[i];
1272           if (sm->num_workers > 1)
1273             {
1274               ip4_header_t ip = {
1275                 .src_address = locals[i].addr,
1276               };
1277               bitmap =
1278                 clib_bitmap_set (bitmap,
1279                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1280             }
1281         }
1282
1283       /* Assign workers */
1284       if (sm->num_workers > 1)
1285         {
1286           /* *INDENT-OFF* */
1287           clib_bitmap_foreach (i, bitmap,
1288             ({
1289                vec_add1(m->workers, i);
1290             }));
1291           /* *INDENT-ON* */
1292         }
1293     }
1294   else
1295     {
1296       if (!m)
1297         return VNET_API_ERROR_NO_SUCH_ENTRY;
1298
1299       if (!is_lb_static_mapping (m))
1300         return VNET_API_ERROR_INVALID_VALUE;
1301
1302       /* Free external address port */
1303       if (!(sm->static_mapping_only || out2in_only))
1304         {
1305           for (i = 0; i < vec_len (sm->addresses); i++)
1306             {
1307               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1308                 {
1309                   a = sm->addresses + i;
1310                   switch (proto)
1311                     {
1312 #define _(N, j, n, s) \
1313                     case SNAT_PROTOCOL_##N: \
1314                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1315                       if (e_port > 1024) \
1316                         { \
1317                           a->busy_##n##_ports--; \
1318                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1319                         } \
1320                       break;
1321                       foreach_snat_protocol
1322 #undef _
1323                     default:
1324                       nat_log_info ("unknown protocol");
1325                       return VNET_API_ERROR_INVALID_VALUE_2;
1326                     }
1327                   break;
1328                 }
1329             }
1330         }
1331
1332       m_key.addr = m->external_addr;
1333       m_key.port = m->external_port;
1334       m_key.protocol = m->proto;
1335       m_key.fib_index = 0;
1336       kv.key = m_key.as_u64;
1337       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1338         {
1339           nat_log_err ("static_mapping_by_external key del failed");
1340           return VNET_API_ERROR_UNSPECIFIED;
1341         }
1342
1343       /* *INDENT-OFF* */
1344       pool_foreach (local, m->locals,
1345       ({
1346           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1347                             FIB_SOURCE_PLUGIN_LOW);
1348           m_key.addr = local->addr;
1349           if (!out2in_only)
1350             {
1351               m_key.port = local->port;
1352               m_key.fib_index = local->fib_index;
1353               kv.key = m_key.as_u64;
1354               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1355                 {
1356                   nat_log_err ("static_mapping_by_local key del failed");
1357                   return VNET_API_ERROR_UNSPECIFIED;
1358                 }
1359             }
1360
1361           if (sm->num_workers > 1)
1362             {
1363               ip4_header_t ip = {
1364                 .src_address = local->addr,
1365               };
1366               tsm = vec_elt_at_index (sm->per_thread_data,
1367                                       sm->worker_in2out_cb (&ip, m->fib_index));
1368             }
1369           else
1370             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1371
1372           /* Delete sessions */
1373           u_key.addr = local->addr;
1374           u_key.fib_index = local->fib_index;
1375           kv.key = u_key.as_u64;
1376           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1377             {
1378               u = pool_elt_at_index (tsm->users, value.value);
1379               if (u->nstaticsessions)
1380                 {
1381                   head_index = u->sessions_per_user_list_head_index;
1382                   head = pool_elt_at_index (tsm->list_pool, head_index);
1383                   elt_index = head->next;
1384                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1385                   ses_index = elt->value;
1386                   while (ses_index != ~0)
1387                     {
1388                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1389                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1390                       ses_index = elt->value;
1391
1392                       if (!(is_lb_session (s)))
1393                         continue;
1394
1395                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1396                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1397                         continue;
1398
1399                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1400                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1401                     }
1402                 }
1403             }
1404       }));
1405       /* *INDENT-ON* */
1406       if (m->affinity)
1407         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1408       pool_free (m->locals);
1409       vec_free (m->tag);
1410       vec_free (m->workers);
1411
1412       pool_put (sm->static_mappings, m);
1413     }
1414
1415   return 0;
1416 }
1417
1418 int
1419 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1420                                        ip4_address_t l_addr, u16 l_port,
1421                                        snat_protocol_t proto, u32 vrf_id,
1422                                        u8 probability, u8 is_add)
1423 {
1424   snat_main_t *sm = &snat_main;
1425   snat_static_mapping_t *m = 0;
1426   snat_session_key_t m_key;
1427   clib_bihash_kv_8_8_t kv, value;
1428   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1429   snat_main_per_thread_data_t *tsm;
1430   snat_user_key_t u_key;
1431   snat_user_t *u;
1432   snat_session_t *s;
1433   dlist_elt_t *head, *elt;
1434   u32 elt_index, head_index, ses_index, *locals = 0;
1435   uword *bitmap = 0;
1436   int i;
1437
1438   if (!sm->endpoint_dependent)
1439     return VNET_API_ERROR_FEATURE_DISABLED;
1440
1441   m_key.addr = e_addr;
1442   m_key.port = e_port;
1443   m_key.protocol = proto;
1444   m_key.fib_index = 0;
1445   kv.key = m_key.as_u64;
1446   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1447     m = pool_elt_at_index (sm->static_mappings, value.value);
1448
1449   if (!m)
1450     return VNET_API_ERROR_NO_SUCH_ENTRY;
1451
1452   if (!is_lb_static_mapping (m))
1453     return VNET_API_ERROR_INVALID_VALUE;
1454
1455   /* *INDENT-OFF* */
1456   pool_foreach (local, m->locals,
1457   ({
1458     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1459         (local->vrf_id == vrf_id))
1460       {
1461         match_local = local;
1462         break;
1463       }
1464   }));
1465   /* *INDENT-ON* */
1466
1467   if (is_add)
1468     {
1469       if (match_local)
1470         return VNET_API_ERROR_VALUE_EXIST;
1471
1472       pool_get (m->locals, local);
1473       clib_memset (local, 0, sizeof (*local));
1474       local->addr.as_u32 = l_addr.as_u32;
1475       local->port = l_port;
1476       local->probability = probability;
1477       local->vrf_id = vrf_id;
1478       local->fib_index =
1479         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1480                                            FIB_SOURCE_PLUGIN_LOW);
1481
1482       if (!is_out2in_only_static_mapping (m))
1483         {
1484           m_key.addr = l_addr;
1485           m_key.port = l_port;
1486           m_key.fib_index = local->fib_index;
1487           kv.key = m_key.as_u64;
1488           kv.value = m - sm->static_mappings;
1489           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1490             nat_log_err ("static_mapping_by_local key add failed");
1491         }
1492     }
1493   else
1494     {
1495       if (!match_local)
1496         return VNET_API_ERROR_NO_SUCH_ENTRY;
1497
1498       if (pool_elts (m->locals) < 3)
1499         return VNET_API_ERROR_UNSPECIFIED;
1500
1501       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1502                         FIB_SOURCE_PLUGIN_LOW);
1503
1504       if (!is_out2in_only_static_mapping (m))
1505         {
1506           m_key.addr = l_addr;
1507           m_key.port = l_port;
1508           m_key.fib_index = match_local->fib_index;
1509           kv.key = m_key.as_u64;
1510           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1511             nat_log_err ("static_mapping_by_local key del failed");
1512         }
1513
1514       if (sm->num_workers > 1)
1515         {
1516           ip4_header_t ip = {
1517             .src_address = local->addr,
1518           };
1519           tsm = vec_elt_at_index (sm->per_thread_data,
1520                                   sm->worker_in2out_cb (&ip, m->fib_index));
1521         }
1522       else
1523         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1524
1525       /* Delete sessions */
1526       u_key.addr = match_local->addr;
1527       u_key.fib_index = match_local->fib_index;
1528       kv.key = u_key.as_u64;
1529       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1530         {
1531           u = pool_elt_at_index (tsm->users, value.value);
1532           if (u->nstaticsessions)
1533             {
1534               head_index = u->sessions_per_user_list_head_index;
1535               head = pool_elt_at_index (tsm->list_pool, head_index);
1536               elt_index = head->next;
1537               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1538               ses_index = elt->value;
1539               while (ses_index != ~0)
1540                 {
1541                   s = pool_elt_at_index (tsm->sessions, ses_index);
1542                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1543                   ses_index = elt->value;
1544
1545                   if (!(is_lb_session (s)))
1546                     continue;
1547
1548                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1549                       (clib_net_to_host_u16 (s->in2out.port) !=
1550                        match_local->port))
1551                     continue;
1552
1553                   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1554                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1555                 }
1556             }
1557         }
1558
1559       pool_put (m->locals, match_local);
1560     }
1561
1562   vec_free (m->workers);
1563
1564   /* *INDENT-OFF* */
1565   pool_foreach (local, m->locals,
1566   ({
1567     vec_add1 (locals, local - m->locals);
1568     if (sm->num_workers > 1)
1569       {
1570         ip4_header_t ip;
1571         ip.src_address.as_u32 = local->addr.as_u32,
1572         bitmap = clib_bitmap_set (bitmap,
1573                                   sm->worker_in2out_cb (&ip, local->fib_index),
1574                                   1);
1575       }
1576   }));
1577   /* *INDENT-ON* */
1578
1579   ASSERT (vec_len (locals) > 1);
1580
1581   local = pool_elt_at_index (m->locals, locals[0]);
1582   local->prefix = local->probability;
1583   for (i = 1; i < vec_len (locals); i++)
1584     {
1585       local = pool_elt_at_index (m->locals, locals[i]);
1586       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1587       local->prefix = local->probability + prev_local->prefix;
1588     }
1589
1590   /* Assign workers */
1591   if (sm->num_workers > 1)
1592     {
1593       /* *INDENT-OFF* */
1594       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1595       /* *INDENT-ON* */
1596     }
1597
1598   return 0;
1599 }
1600
1601 int
1602 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1603                   u8 twice_nat)
1604 {
1605   snat_address_t *a = 0;
1606   snat_session_t *ses;
1607   u32 *ses_to_be_removed = 0, *ses_index;
1608   snat_main_per_thread_data_t *tsm;
1609   snat_static_mapping_t *m;
1610   snat_interface_t *interface;
1611   int i;
1612   snat_address_t *addresses =
1613     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1614
1615   /* Find SNAT address */
1616   for (i = 0; i < vec_len (addresses); i++)
1617     {
1618       if (addresses[i].addr.as_u32 == addr.as_u32)
1619         {
1620           a = addresses + i;
1621           break;
1622         }
1623     }
1624   if (!a)
1625     return VNET_API_ERROR_NO_SUCH_ENTRY;
1626
1627   if (delete_sm)
1628     {
1629       /* *INDENT-OFF* */
1630       pool_foreach (m, sm->static_mappings,
1631       ({
1632           if (m->external_addr.as_u32 == addr.as_u32)
1633             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1634                                             m->local_port, m->external_port,
1635                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1636                                             m->proto, 0, m->twice_nat,
1637                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1638       }));
1639       /* *INDENT-ON* */
1640     }
1641   else
1642     {
1643       /* Check if address is used in some static mapping */
1644       if (is_snat_address_used_in_static_mapping (sm, addr))
1645         {
1646           nat_log_notice ("address used in static mapping");
1647           return VNET_API_ERROR_UNSPECIFIED;
1648         }
1649     }
1650
1651   if (a->fib_index != ~0)
1652     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1653
1654   /* Delete sessions using address */
1655   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1656     {
1657       /* *INDENT-OFF* */
1658       vec_foreach (tsm, sm->per_thread_data)
1659         {
1660           pool_foreach (ses, tsm->sessions, ({
1661             if (ses->out2in.addr.as_u32 == addr.as_u32)
1662               {
1663                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1664                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1665               }
1666           }));
1667
1668           vec_foreach (ses_index, ses_to_be_removed)
1669             {
1670               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1671               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1672             }
1673
1674           vec_free (ses_to_be_removed);
1675         }
1676       /* *INDENT-ON* */
1677     }
1678
1679 #define _(N, i, n, s) \
1680   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1681   vec_free (a->busy_##n##_ports_per_thread);
1682   foreach_snat_protocol
1683 #undef _
1684     if (twice_nat)
1685     {
1686       vec_del1 (sm->twice_nat_addresses, i);
1687       return 0;
1688     }
1689   else
1690     vec_del1 (sm->addresses, i);
1691
1692   /* Delete external address from FIB */
1693   /* *INDENT-OFF* */
1694   pool_foreach (interface, sm->interfaces,
1695   ({
1696     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1697       continue;
1698
1699     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1700     break;
1701   }));
1702   pool_foreach (interface, sm->output_feature_interfaces,
1703   ({
1704     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1705       continue;
1706
1707     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1708     break;
1709   }));
1710   /* *INDENT-ON* */
1711
1712   return 0;
1713 }
1714
1715 int
1716 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1717 {
1718   snat_main_t *sm = &snat_main;
1719   snat_interface_t *i;
1720   const char *feature_name, *del_feature_name;
1721   snat_address_t *ap;
1722   snat_static_mapping_t *m;
1723   snat_det_map_t *dm;
1724   nat_outside_fib_t *outside_fib;
1725   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1726                                                        sw_if_index);
1727
1728   if (sm->out2in_dpo && !is_inside)
1729     return VNET_API_ERROR_UNSUPPORTED;
1730
1731   /* *INDENT-OFF* */
1732   pool_foreach (i, sm->output_feature_interfaces,
1733   ({
1734     if (i->sw_if_index == sw_if_index)
1735       return VNET_API_ERROR_VALUE_EXIST;
1736   }));
1737   /* *INDENT-ON* */
1738
1739   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1740     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1741   else
1742     {
1743       if (sm->num_workers > 1 && !sm->deterministic)
1744         feature_name =
1745           is_inside ? "nat44-in2out-worker-handoff" :
1746           "nat44-out2in-worker-handoff";
1747       else if (sm->deterministic)
1748         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1749       else if (sm->endpoint_dependent)
1750         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1751       else
1752         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1753     }
1754
1755   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1756     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1757                                                       NAT_FQ_NELTS);
1758
1759   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1760     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1761                                                       NAT_FQ_NELTS);
1762
1763   if (!is_inside)
1764     {
1765       /* *INDENT-OFF* */
1766       vec_foreach (outside_fib, sm->outside_fibs)
1767         {
1768           if (outside_fib->fib_index == fib_index)
1769             {
1770               if (is_del)
1771                 {
1772                   outside_fib->refcount--;
1773                   if (!outside_fib->refcount)
1774                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1775                 }
1776               else
1777                 outside_fib->refcount++;
1778               goto feature_set;
1779             }
1780         }
1781       /* *INDENT-ON* */
1782       if (!is_del)
1783         {
1784           vec_add2 (sm->outside_fibs, outside_fib, 1);
1785           outside_fib->refcount = 1;
1786           outside_fib->fib_index = fib_index;
1787         }
1788     }
1789 feature_set:
1790   /* *INDENT-OFF* */
1791   pool_foreach (i, sm->interfaces,
1792   ({
1793     if (i->sw_if_index == sw_if_index)
1794       {
1795         if (is_del)
1796           {
1797             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1798               {
1799                 if (is_inside)
1800                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1801                 else
1802                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1803
1804                 if (sm->num_workers > 1 && !sm->deterministic)
1805                   {
1806                     del_feature_name = "nat44-handoff-classify";
1807                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1808                                                  "nat44-out2in-worker-handoff";
1809                   }
1810                 else if (sm->deterministic)
1811                   {
1812                     del_feature_name = "nat44-det-classify";
1813                     feature_name = !is_inside ?  "nat44-det-in2out" :
1814                                                  "nat44-det-out2in";
1815                   }
1816                 else if (sm->endpoint_dependent)
1817                   {
1818                     del_feature_name = "nat44-ed-classify";
1819                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1820                                                  "nat44-ed-out2in";
1821                   }
1822                 else
1823                   {
1824                     del_feature_name = "nat44-classify";
1825                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1826                   }
1827
1828                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1829                                              sw_if_index, 0, 0, 0);
1830                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1831                                              sw_if_index, 1, 0, 0);
1832                 if (!is_inside)
1833                   {
1834                     if (sm->endpoint_dependent)
1835                       vnet_feature_enable_disable ("ip4-local",
1836                                                    "nat44-ed-hairpinning",
1837                                                    sw_if_index, 1, 0, 0);
1838                     else if (!sm->deterministic)
1839                       vnet_feature_enable_disable ("ip4-local",
1840                                                    "nat44-hairpinning",
1841                                                    sw_if_index, 1, 0, 0);
1842                   }
1843               }
1844             else
1845               {
1846                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1847                                              sw_if_index, 0, 0, 0);
1848                 pool_put (sm->interfaces, i);
1849                 if (is_inside)
1850                   {
1851                     if (sm->endpoint_dependent)
1852                       vnet_feature_enable_disable ("ip4-local",
1853                                                    "nat44-ed-hairpinning",
1854                                                    sw_if_index, 0, 0, 0);
1855                     else if (!sm->deterministic)
1856                       vnet_feature_enable_disable ("ip4-local",
1857                                                    "nat44-hairpinning",
1858                                                    sw_if_index, 0, 0, 0);
1859                   }
1860               }
1861           }
1862         else
1863           {
1864             if ((nat_interface_is_inside(i) && is_inside) ||
1865                 (nat_interface_is_outside(i) && !is_inside))
1866               return 0;
1867
1868             if (sm->num_workers > 1 && !sm->deterministic)
1869               {
1870                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1871                                                  "nat44-out2in-worker-handoff";
1872                 feature_name = "nat44-handoff-classify";
1873               }
1874             else if (sm->deterministic)
1875               {
1876                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1877                                                  "nat44-det-out2in";
1878                 feature_name = "nat44-det-classify";
1879               }
1880             else if (sm->endpoint_dependent)
1881               {
1882                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1883                                                  "nat44-ed-out2in";
1884                 feature_name = "nat44-ed-classify";
1885               }
1886             else
1887               {
1888                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1889                 feature_name = "nat44-classify";
1890               }
1891
1892             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1893                                          sw_if_index, 0, 0, 0);
1894             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1895                                          sw_if_index, 1, 0, 0);
1896             if (!is_inside)
1897               {
1898                 if (sm->endpoint_dependent)
1899                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1900                                                sw_if_index, 0, 0, 0);
1901                 else if (!sm->deterministic)
1902                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1903                                                sw_if_index, 0, 0, 0);
1904               }
1905             goto set_flags;
1906           }
1907
1908         goto fib;
1909       }
1910   }));
1911   /* *INDENT-ON* */
1912
1913   if (is_del)
1914     return VNET_API_ERROR_NO_SUCH_ENTRY;
1915
1916   pool_get (sm->interfaces, i);
1917   i->sw_if_index = sw_if_index;
1918   i->flags = 0;
1919   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1920                                0);
1921
1922   if (is_inside && !sm->out2in_dpo)
1923     {
1924       if (sm->endpoint_dependent)
1925         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1926                                      sw_if_index, 1, 0, 0);
1927       else if (!sm->deterministic)
1928         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1929                                      sw_if_index, 1, 0, 0);
1930     }
1931
1932 set_flags:
1933   if (is_inside)
1934     {
1935       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1936       return 0;
1937     }
1938   else
1939     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1940
1941   /* Add/delete external addresses to FIB */
1942 fib:
1943   /* *INDENT-OFF* */
1944   vec_foreach (ap, sm->addresses)
1945     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1946
1947   pool_foreach (m, sm->static_mappings,
1948   ({
1949     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1950       continue;
1951
1952     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1953   }));
1954
1955   pool_foreach (dm, sm->det_maps,
1956   ({
1957     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1958   }));
1959   /* *INDENT-ON* */
1960
1961   return 0;
1962 }
1963
1964 int
1965 snat_interface_add_del_output_feature (u32 sw_if_index,
1966                                        u8 is_inside, int is_del)
1967 {
1968   snat_main_t *sm = &snat_main;
1969   snat_interface_t *i;
1970   snat_address_t *ap;
1971   snat_static_mapping_t *m;
1972   nat_outside_fib_t *outside_fib;
1973   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1974                                                        sw_if_index);
1975
1976
1977   if (sm->deterministic ||
1978       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1979     return VNET_API_ERROR_UNSUPPORTED;
1980
1981   /* *INDENT-OFF* */
1982   pool_foreach (i, sm->interfaces,
1983   ({
1984     if (i->sw_if_index == sw_if_index)
1985       return VNET_API_ERROR_VALUE_EXIST;
1986   }));
1987   /* *INDENT-ON* */
1988
1989   if (!is_inside)
1990     {
1991       /* *INDENT-OFF* */
1992       vec_foreach (outside_fib, sm->outside_fibs)
1993         {
1994           if (outside_fib->fib_index == fib_index)
1995             {
1996               if (is_del)
1997                 {
1998                   outside_fib->refcount--;
1999                   if (!outside_fib->refcount)
2000                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2001                 }
2002               else
2003                 outside_fib->refcount++;
2004               goto feature_set;
2005             }
2006         }
2007       /* *INDENT-ON* */
2008       if (!is_del)
2009         {
2010           vec_add2 (sm->outside_fibs, outside_fib, 1);
2011           outside_fib->refcount = 1;
2012           outside_fib->fib_index = fib_index;
2013         }
2014     }
2015
2016 feature_set:
2017   if (is_inside)
2018     {
2019       if (sm->endpoint_dependent)
2020         {
2021           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2022                                        sw_if_index, !is_del, 0, 0);
2023           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2024                                        sw_if_index, !is_del, 0, 0);
2025         }
2026       else
2027         {
2028           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2029                                        sw_if_index, !is_del, 0, 0);
2030           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2031                                        sw_if_index, !is_del, 0, 0);
2032         }
2033       goto fq;
2034     }
2035
2036   if (sm->num_workers > 1)
2037     {
2038       vnet_feature_enable_disable ("ip4-unicast",
2039                                    "nat44-out2in-worker-handoff",
2040                                    sw_if_index, !is_del, 0, 0);
2041       vnet_feature_enable_disable ("ip4-output",
2042                                    "nat44-in2out-output-worker-handoff",
2043                                    sw_if_index, !is_del, 0, 0);
2044     }
2045   else
2046     {
2047       if (sm->endpoint_dependent)
2048         {
2049           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2050                                        sw_if_index, !is_del, 0, 0);
2051           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2052                                        sw_if_index, !is_del, 0, 0);
2053         }
2054       else
2055         {
2056           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2057                                        sw_if_index, !is_del, 0, 0);
2058           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2059                                        sw_if_index, !is_del, 0, 0);
2060         }
2061     }
2062
2063 fq:
2064   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2065     sm->fq_in2out_output_index =
2066       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2067
2068   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2069     sm->fq_out2in_index =
2070       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2071
2072   /* *INDENT-OFF* */
2073   pool_foreach (i, sm->output_feature_interfaces,
2074   ({
2075     if (i->sw_if_index == sw_if_index)
2076       {
2077         if (is_del)
2078           pool_put (sm->output_feature_interfaces, i);
2079         else
2080           return VNET_API_ERROR_VALUE_EXIST;
2081
2082         goto fib;
2083       }
2084   }));
2085   /* *INDENT-ON* */
2086
2087   if (is_del)
2088     return VNET_API_ERROR_NO_SUCH_ENTRY;
2089
2090   pool_get (sm->output_feature_interfaces, i);
2091   i->sw_if_index = sw_if_index;
2092   i->flags = 0;
2093   if (is_inside)
2094     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2095   else
2096     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2097
2098   /* Add/delete external addresses to FIB */
2099 fib:
2100   if (is_inside)
2101     return 0;
2102
2103   /* *INDENT-OFF* */
2104   vec_foreach (ap, sm->addresses)
2105     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2106
2107   pool_foreach (m, sm->static_mappings,
2108   ({
2109     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2110       continue;
2111
2112     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2113   }));
2114   /* *INDENT-ON* */
2115
2116   return 0;
2117 }
2118
2119 int
2120 snat_set_workers (uword * bitmap)
2121 {
2122   snat_main_t *sm = &snat_main;
2123   int i, j = 0;
2124
2125   if (sm->num_workers < 2)
2126     return VNET_API_ERROR_FEATURE_DISABLED;
2127
2128   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2129     return VNET_API_ERROR_INVALID_WORKER;
2130
2131   vec_free (sm->workers);
2132   /* *INDENT-OFF* */
2133   clib_bitmap_foreach (i, bitmap,
2134     ({
2135       vec_add1(sm->workers, i);
2136       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2137       j++;
2138     }));
2139   /* *INDENT-ON* */
2140
2141   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2142   sm->num_snat_thread = _vec_len (sm->workers);
2143
2144   return 0;
2145 }
2146
2147
2148 static void
2149 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2150                                        uword opaque,
2151                                        u32 sw_if_index,
2152                                        ip4_address_t * address,
2153                                        u32 address_length,
2154                                        u32 if_address_index, u32 is_delete);
2155
2156 static void
2157 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2158                                  uword opaque,
2159                                  u32 sw_if_index,
2160                                  ip4_address_t * address,
2161                                  u32 address_length,
2162                                  u32 if_address_index, u32 is_delete);
2163
2164 static int
2165 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2166                                  u32 fib_index,
2167                                  u32 thread_index,
2168                                  snat_session_key_t * k,
2169                                  u16 port_per_thread, u32 snat_thread_index);
2170
2171 static clib_error_t *
2172 snat_init (vlib_main_t * vm)
2173 {
2174   snat_main_t *sm = &snat_main;
2175   clib_error_t *error = 0;
2176   ip4_main_t *im = &ip4_main;
2177   ip_lookup_main_t *lm = &im->lookup_main;
2178   uword *p;
2179   vlib_thread_registration_t *tr;
2180   vlib_thread_main_t *tm = vlib_get_thread_main ();
2181   uword *bitmap = 0;
2182   u32 i;
2183   ip4_add_del_interface_address_callback_t cb4;
2184   vlib_node_t *error_drop_node;
2185
2186   sm->vlib_main = vm;
2187   sm->vnet_main = vnet_get_main ();
2188   sm->ip4_main = im;
2189   sm->ip4_lookup_main = lm;
2190   sm->api_main = &api_main;
2191   sm->first_worker_index = 0;
2192   sm->num_workers = 0;
2193   sm->num_snat_thread = 1;
2194   sm->workers = 0;
2195   sm->port_per_thread = 0xffff - 1024;
2196   sm->fq_in2out_index = ~0;
2197   sm->fq_out2in_index = ~0;
2198   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2199   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2200   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2201   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2202   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2203   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2204   sm->forwarding_enabled = 0;
2205   sm->log_class = vlib_log_register_class ("nat", 0);
2206   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2207   sm->error_node_index = error_drop_node->index;
2208   sm->mss_clamping = 0;
2209
2210   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2211   if (p)
2212     {
2213       tr = (vlib_thread_registration_t *) p[0];
2214       if (tr)
2215         {
2216           sm->num_workers = tr->count;
2217           sm->first_worker_index = tr->first_index;
2218         }
2219     }
2220
2221   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2222
2223   /* Use all available workers by default */
2224   if (sm->num_workers > 1)
2225     {
2226       for (i = 0; i < sm->num_workers; i++)
2227         bitmap = clib_bitmap_set (bitmap, i, 1);
2228       snat_set_workers (bitmap);
2229       clib_bitmap_free (bitmap);
2230     }
2231   else
2232     {
2233       sm->per_thread_data[0].snat_thread_index = 0;
2234     }
2235
2236   error = snat_api_init (vm, sm);
2237   if (error)
2238     return error;
2239
2240   /* Set up the interface address add/del callback */
2241   cb4.function = snat_ip4_add_del_interface_address_cb;
2242   cb4.function_opaque = 0;
2243
2244   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2245
2246   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2247   cb4.function_opaque = 0;
2248
2249   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2250
2251   nat_dpo_module_init ();
2252
2253   /* Init counters */
2254   sm->total_users.name = "total-users";
2255   sm->total_users.stat_segment_name = "/nat44/total-users";
2256   vlib_validate_simple_counter (&sm->total_users, 0);
2257   vlib_zero_simple_counter (&sm->total_users, 0);
2258   sm->total_sessions.name = "total-sessions";
2259   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2260   vlib_validate_simple_counter (&sm->total_sessions, 0);
2261   vlib_zero_simple_counter (&sm->total_sessions, 0);
2262
2263   /* Init IPFIX logging */
2264   snat_ipfix_logging_init (vm);
2265
2266   /* Init NAT64 */
2267   error = nat64_init (vm);
2268   if (error)
2269     return error;
2270
2271   dslite_init (vm);
2272
2273   nat66_init ();
2274
2275   /* Init virtual fragmenentation reassembly */
2276   return nat_reass_init (vm);
2277 }
2278
2279 VLIB_INIT_FUNCTION (snat_init);
2280
2281 void
2282 snat_free_outside_address_and_port (snat_address_t * addresses,
2283                                     u32 thread_index, snat_session_key_t * k)
2284 {
2285   snat_address_t *a;
2286   u32 address_index;
2287   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2288
2289   for (address_index = 0; address_index < vec_len (addresses);
2290        address_index++)
2291     {
2292       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2293         break;
2294     }
2295
2296   ASSERT (address_index < vec_len (addresses));
2297
2298   a = addresses + address_index;
2299
2300   switch (k->protocol)
2301     {
2302 #define _(N, i, n, s) \
2303     case SNAT_PROTOCOL_##N: \
2304       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2305         port_host_byte_order) == 1); \
2306       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2307         port_host_byte_order, 0); \
2308       a->busy_##n##_ports--; \
2309       a->busy_##n##_ports_per_thread[thread_index]--; \
2310       break;
2311       foreach_snat_protocol
2312 #undef _
2313     default:
2314       nat_log_info ("unknown protocol");
2315       return;
2316     }
2317 }
2318
2319 int
2320 snat_static_mapping_match (snat_main_t * sm,
2321                            snat_session_key_t match,
2322                            snat_session_key_t * mapping,
2323                            u8 by_external,
2324                            u8 * is_addr_only,
2325                            twice_nat_type_t * twice_nat,
2326                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2327                            u8 * is_identity_nat)
2328 {
2329   clib_bihash_kv_8_8_t kv, value;
2330   snat_static_mapping_t *m;
2331   snat_session_key_t m_key;
2332   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2333   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2334   u8 backend_index;
2335   nat44_lb_addr_port_t *local;
2336
2337   m_key.fib_index = match.fib_index;
2338   if (by_external)
2339     {
2340       mapping_hash = &sm->static_mapping_by_external;
2341       m_key.fib_index = 0;
2342     }
2343
2344   m_key.addr = match.addr;
2345   m_key.port = clib_net_to_host_u16 (match.port);
2346   m_key.protocol = match.protocol;
2347
2348   kv.key = m_key.as_u64;
2349
2350   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2351     {
2352       /* Try address only mapping */
2353       m_key.port = 0;
2354       m_key.protocol = 0;
2355       kv.key = m_key.as_u64;
2356       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2357         return 1;
2358     }
2359
2360   m = pool_elt_at_index (sm->static_mappings, value.value);
2361
2362   if (by_external)
2363     {
2364       if (is_lb_static_mapping (m))
2365         {
2366           if (PREDICT_FALSE (lb != 0))
2367             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2368           if (m->affinity)
2369             {
2370               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2371                                               match.protocol, match.port,
2372                                               &backend_index))
2373                 goto get_local;
2374
2375               local = pool_elt_at_index (m->locals, backend_index);
2376               mapping->addr = local->addr;
2377               mapping->port = clib_host_to_net_u16 (local->port);
2378               mapping->fib_index = local->fib_index;
2379               goto end;
2380             }
2381         get_local:
2382           /* *INDENT-OFF* */
2383           pool_foreach_index (i, m->locals,
2384           ({
2385             vec_add1 (tmp, i);
2386           }));
2387           /* *INDENT-ON* */
2388           hi = vec_len (tmp) - 1;
2389           local = pool_elt_at_index (m->locals, tmp[hi]);
2390           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2391           while (lo < hi)
2392             {
2393               mid = ((hi - lo) >> 1) + lo;
2394               local = pool_elt_at_index (m->locals, tmp[mid]);
2395               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2396             }
2397           local = pool_elt_at_index (m->locals, tmp[lo]);
2398           if (!(local->prefix >= rand))
2399             return 1;
2400           if (PREDICT_FALSE (sm->num_workers > 1))
2401             {
2402               ip4_header_t ip = {
2403                 .src_address = local->addr,
2404               };
2405               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2406                   vlib_get_thread_index ())
2407                 goto get_local;
2408             }
2409           mapping->addr = local->addr;
2410           mapping->port = clib_host_to_net_u16 (local->port);
2411           mapping->fib_index = local->fib_index;
2412           if (m->affinity)
2413             {
2414               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2415                                                 match.protocol, match.port,
2416                                                 tmp[lo], m->affinity,
2417                                                 m->affinity_per_service_list_head_index))
2418                 nat_log_info ("create affinity record failed");
2419             }
2420           vec_free (tmp);
2421         }
2422       else
2423         {
2424           if (PREDICT_FALSE (lb != 0))
2425             *lb = NO_LB_NAT;
2426           mapping->fib_index = m->fib_index;
2427           mapping->addr = m->local_addr;
2428           /* Address only mapping doesn't change port */
2429           mapping->port = is_addr_only_static_mapping (m) ? match.port
2430             : clib_host_to_net_u16 (m->local_port);
2431         }
2432       mapping->protocol = m->proto;
2433     }
2434   else
2435     {
2436       mapping->addr = m->external_addr;
2437       /* Address only mapping doesn't change port */
2438       mapping->port = is_addr_only_static_mapping (m) ? match.port
2439         : clib_host_to_net_u16 (m->external_port);
2440       mapping->fib_index = sm->outside_fib_index;
2441     }
2442
2443 end:
2444   if (PREDICT_FALSE (is_addr_only != 0))
2445     *is_addr_only = is_addr_only_static_mapping (m);
2446
2447   if (PREDICT_FALSE (twice_nat != 0))
2448     *twice_nat = m->twice_nat;
2449
2450   if (PREDICT_FALSE (is_identity_nat != 0))
2451     *is_identity_nat = is_identity_static_mapping (m);
2452
2453   return 0;
2454 }
2455
2456 static_always_inline u16
2457 snat_random_port (u16 min, u16 max)
2458 {
2459   snat_main_t *sm = &snat_main;
2460   return min + random_u32 (&sm->random_seed) /
2461     (random_u32_max () / (max - min + 1) + 1);
2462 }
2463
2464 int
2465 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2466                                      u32 fib_index,
2467                                      u32 thread_index,
2468                                      snat_session_key_t * k,
2469                                      u16 port_per_thread,
2470                                      u32 snat_thread_index)
2471 {
2472   snat_main_t *sm = &snat_main;
2473
2474   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2475                                   port_per_thread, snat_thread_index);
2476 }
2477
2478 static int
2479 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2480                                  u32 fib_index,
2481                                  u32 thread_index,
2482                                  snat_session_key_t * k,
2483                                  u16 port_per_thread, u32 snat_thread_index)
2484 {
2485   int i;
2486   snat_address_t *a, *ga = 0;
2487   u32 portnum;
2488
2489   for (i = 0; i < vec_len (addresses); i++)
2490     {
2491       a = addresses + i;
2492       switch (k->protocol)
2493         {
2494 #define _(N, j, n, s) \
2495         case SNAT_PROTOCOL_##N: \
2496           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2497             { \
2498               if (a->fib_index == fib_index) \
2499                 { \
2500                   while (1) \
2501                     { \
2502                       portnum = (port_per_thread * \
2503                         snat_thread_index) + \
2504                         snat_random_port(1, port_per_thread) + 1024; \
2505                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2506                         continue; \
2507                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2508                       a->busy_##n##_ports_per_thread[thread_index]++; \
2509                       a->busy_##n##_ports++; \
2510                       k->addr = a->addr; \
2511                       k->port = clib_host_to_net_u16(portnum); \
2512                       return 0; \
2513                     } \
2514                 } \
2515               else if (a->fib_index == ~0) \
2516                 { \
2517                   ga = a; \
2518                 } \
2519             } \
2520           break;
2521           foreach_snat_protocol
2522 #undef _
2523         default:
2524           nat_log_info ("unknown protocol");
2525           return 1;
2526         }
2527
2528     }
2529
2530   if (ga)
2531     {
2532       a = ga;
2533       switch (k->protocol)
2534         {
2535 #define _(N, j, n, s) \
2536         case SNAT_PROTOCOL_##N: \
2537           while (1) \
2538             { \
2539               portnum = (port_per_thread * \
2540                 snat_thread_index) + \
2541                 snat_random_port(1, port_per_thread) + 1024; \
2542               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2543                 continue; \
2544               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2545               a->busy_##n##_ports_per_thread[thread_index]++; \
2546               a->busy_##n##_ports++; \
2547               k->addr = a->addr; \
2548               k->port = clib_host_to_net_u16(portnum); \
2549               return 0; \
2550             }
2551           break;
2552           foreach_snat_protocol
2553 #undef _
2554         default:
2555           nat_log_info ("unknown protocol");
2556           return 1;
2557         }
2558     }
2559
2560   /* Totally out of translations to use... */
2561   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2562   return 1;
2563 }
2564
2565 static int
2566 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2567                               u32 fib_index,
2568                               u32 thread_index,
2569                               snat_session_key_t * k,
2570                               u16 port_per_thread, u32 snat_thread_index)
2571 {
2572   snat_main_t *sm = &snat_main;
2573   snat_address_t *a = addresses;
2574   u16 m, ports, portnum, A, j;
2575   m = 16 - (sm->psid_offset + sm->psid_length);
2576   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2577
2578   if (!vec_len (addresses))
2579     goto exhausted;
2580
2581   switch (k->protocol)
2582     {
2583 #define _(N, i, n, s) \
2584     case SNAT_PROTOCOL_##N: \
2585       if (a->busy_##n##_ports < ports) \
2586         { \
2587           while (1) \
2588             { \
2589               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2590               j = snat_random_port(0, pow2_mask(m)); \
2591               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2592               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2593                 continue; \
2594               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2595               a->busy_##n##_ports++; \
2596               k->addr = a->addr; \
2597               k->port = clib_host_to_net_u16 (portnum); \
2598               return 0; \
2599             } \
2600         } \
2601       break;
2602       foreach_snat_protocol
2603 #undef _
2604     default:
2605       nat_log_info ("unknown protocol");
2606       return 1;
2607     }
2608
2609 exhausted:
2610   /* Totally out of translations to use... */
2611   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2612   return 1;
2613 }
2614
2615 static int
2616 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2617                                u32 fib_index,
2618                                u32 thread_index,
2619                                snat_session_key_t * k,
2620                                u16 port_per_thread, u32 snat_thread_index)
2621 {
2622   snat_main_t *sm = &snat_main;
2623   snat_address_t *a = addresses;
2624   u16 portnum, ports;
2625
2626   ports = sm->end_port - sm->start_port + 1;
2627
2628   if (!vec_len (addresses))
2629     goto exhausted;
2630
2631   switch (k->protocol)
2632     {
2633 #define _(N, i, n, s) \
2634     case SNAT_PROTOCOL_##N: \
2635       if (a->busy_##n##_ports < ports) \
2636         { \
2637           while (1) \
2638             { \
2639               portnum = snat_random_port(sm->start_port, sm->end_port); \
2640               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2641                 continue; \
2642               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2643               a->busy_##n##_ports++; \
2644               k->addr = a->addr; \
2645               k->port = clib_host_to_net_u16 (portnum); \
2646               return 0; \
2647             } \
2648         } \
2649       break;
2650       foreach_snat_protocol
2651 #undef _
2652     default:
2653       nat_log_info ("unknown protocol");
2654       return 1;
2655     }
2656
2657 exhausted:
2658   /* Totally out of translations to use... */
2659   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2660   return 1;
2661 }
2662
2663 void
2664 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2665 {
2666   dpo_id_t dpo_v4 = DPO_INVALID;
2667   fib_prefix_t pfx = {
2668     .fp_proto = FIB_PROTOCOL_IP4,
2669     .fp_len = 32,
2670     .fp_addr.ip4.as_u32 = addr.as_u32,
2671   };
2672
2673   if (is_add)
2674     {
2675       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2676       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2677                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2678       dpo_reset (&dpo_v4);
2679     }
2680   else
2681     {
2682       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2683     }
2684 }
2685
2686 u8 *
2687 format_session_kvp (u8 * s, va_list * args)
2688 {
2689   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2690   snat_session_key_t k;
2691
2692   k.as_u64 = v->key;
2693
2694   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2695
2696   return s;
2697 }
2698
2699 u8 *
2700 format_static_mapping_kvp (u8 * s, va_list * args)
2701 {
2702   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2703   snat_session_key_t k;
2704
2705   k.as_u64 = v->key;
2706
2707   s = format (s, "%U static-mapping-index %llu",
2708               format_static_mapping_key, &k, v->value);
2709
2710   return s;
2711 }
2712
2713 u8 *
2714 format_user_kvp (u8 * s, va_list * args)
2715 {
2716   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2717   snat_user_key_t k;
2718
2719   k.as_u64 = v->key;
2720
2721   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2722               k.fib_index, v->value);
2723
2724   return s;
2725 }
2726
2727 u8 *
2728 format_ed_session_kvp (u8 * s, va_list * args)
2729 {
2730   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2731   nat_ed_ses_key_t k;
2732
2733   k.as_u64[0] = v->key[0];
2734   k.as_u64[1] = v->key[1];
2735
2736   s =
2737     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2738             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2739             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2740             format_ip_protocol, k.proto, k.fib_index, v->value);
2741
2742   return s;
2743 }
2744
2745 static u32
2746 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2747 {
2748   snat_main_t *sm = &snat_main;
2749   u32 next_worker_index = 0;
2750   u32 hash;
2751
2752   next_worker_index = sm->first_worker_index;
2753   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2754     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2755
2756   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2757     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2758   else
2759     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2760
2761   return next_worker_index;
2762 }
2763
2764 static u32
2765 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2766 {
2767   snat_main_t *sm = &snat_main;
2768   udp_header_t *udp;
2769   u16 port;
2770   snat_session_key_t m_key;
2771   clib_bihash_kv_8_8_t kv, value;
2772   snat_static_mapping_t *m;
2773   u32 proto;
2774   u32 next_worker_index = 0;
2775
2776   /* first try static mappings without port */
2777   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2778     {
2779       m_key.addr = ip0->dst_address;
2780       m_key.port = 0;
2781       m_key.protocol = 0;
2782       m_key.fib_index = rx_fib_index0;
2783       kv.key = m_key.as_u64;
2784       if (!clib_bihash_search_8_8
2785           (&sm->static_mapping_by_external, &kv, &value))
2786         {
2787           m = pool_elt_at_index (sm->static_mappings, value.value);
2788           return m->workers[0];
2789         }
2790     }
2791
2792   proto = ip_proto_to_snat_proto (ip0->protocol);
2793   udp = ip4_next_header (ip0);
2794   port = udp->dst_port;
2795
2796   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2797     {
2798       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2799         return vlib_get_thread_index ();
2800
2801       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2802         {
2803           nat_reass_ip4_t *reass;
2804
2805           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2806                                       ip0->fragment_id, ip0->protocol);
2807
2808           if (reass && (reass->thread_index != (u32) ~ 0))
2809             return reass->thread_index;
2810           else
2811             return vlib_get_thread_index ();
2812         }
2813     }
2814
2815   /* unknown protocol */
2816   if (PREDICT_FALSE (proto == ~0))
2817     {
2818       /* use current thread */
2819       return vlib_get_thread_index ();
2820     }
2821
2822   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2823     {
2824       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2825       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2826       if (!icmp_is_error_message (icmp))
2827         port = echo->identifier;
2828       else
2829         {
2830           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2831           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2832           void *l4_header = ip4_next_header (inner_ip);
2833           switch (proto)
2834             {
2835             case SNAT_PROTOCOL_ICMP:
2836               icmp = (icmp46_header_t *) l4_header;
2837               echo = (icmp_echo_header_t *) (icmp + 1);
2838               port = echo->identifier;
2839               break;
2840             case SNAT_PROTOCOL_UDP:
2841             case SNAT_PROTOCOL_TCP:
2842               port = ((tcp_udp_header_t *) l4_header)->src_port;
2843               break;
2844             default:
2845               return vlib_get_thread_index ();
2846             }
2847         }
2848     }
2849
2850   /* try static mappings with port */
2851   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2852     {
2853       m_key.addr = ip0->dst_address;
2854       m_key.port = clib_net_to_host_u16 (port);
2855       m_key.protocol = proto;
2856       m_key.fib_index = rx_fib_index0;
2857       kv.key = m_key.as_u64;
2858       if (!clib_bihash_search_8_8
2859           (&sm->static_mapping_by_external, &kv, &value))
2860         {
2861           m = pool_elt_at_index (sm->static_mappings, value.value);
2862           return m->workers[0];
2863         }
2864     }
2865
2866   /* worker by outside port */
2867   next_worker_index = sm->first_worker_index;
2868   next_worker_index +=
2869     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2870   return next_worker_index;
2871 }
2872
2873 static u32
2874 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2875 {
2876   snat_main_t *sm = &snat_main;
2877   clib_bihash_kv_8_8_t kv, value;
2878   u32 proto, next_worker_index = 0;
2879   udp_header_t *udp;
2880   u16 port;
2881   snat_static_mapping_t *m;
2882   u32 hash;
2883
2884   /* first try static mappings without port */
2885   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2886     {
2887       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2888       if (!clib_bihash_search_8_8
2889           (&sm->static_mapping_by_external, &kv, &value))
2890         {
2891           m = pool_elt_at_index (sm->static_mappings, value.value);
2892           return m->workers[0];
2893         }
2894     }
2895
2896   proto = ip_proto_to_snat_proto (ip->protocol);
2897
2898   /* unknown protocol */
2899   if (PREDICT_FALSE (proto == ~0))
2900     {
2901       /* use current thread */
2902       return vlib_get_thread_index ();
2903     }
2904
2905   udp = ip4_next_header (ip);
2906   port = udp->dst_port;
2907
2908   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2909     {
2910       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2911       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2912       if (!icmp_is_error_message (icmp))
2913         port = echo->identifier;
2914       else
2915         {
2916           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2917           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2918           void *l4_header = ip4_next_header (inner_ip);
2919           switch (proto)
2920             {
2921             case SNAT_PROTOCOL_ICMP:
2922               icmp = (icmp46_header_t *) l4_header;
2923               echo = (icmp_echo_header_t *) (icmp + 1);
2924               port = echo->identifier;
2925               break;
2926             case SNAT_PROTOCOL_UDP:
2927             case SNAT_PROTOCOL_TCP:
2928               port = ((tcp_udp_header_t *) l4_header)->src_port;
2929               break;
2930             default:
2931               return vlib_get_thread_index ();
2932             }
2933         }
2934     }
2935
2936   /* try static mappings with port */
2937   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2938     {
2939       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2940                   clib_net_to_host_u16 (port));
2941       if (!clib_bihash_search_8_8
2942           (&sm->static_mapping_by_external, &kv, &value))
2943         {
2944           m = pool_elt_at_index (sm->static_mappings, value.value);
2945           if (!is_lb_static_mapping (m))
2946             return m->workers[0];
2947
2948           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2949             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
2950
2951           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2952             return m->workers[hash & (_vec_len (m->workers) - 1)];
2953           else
2954             return m->workers[hash % _vec_len (m->workers)];
2955         }
2956     }
2957
2958   /* worker by outside port */
2959   next_worker_index = sm->first_worker_index;
2960   next_worker_index +=
2961     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2962
2963   return next_worker_index;
2964 }
2965
2966 static clib_error_t *
2967 snat_config (vlib_main_t * vm, unformat_input_t * input)
2968 {
2969   snat_main_t *sm = &snat_main;
2970   nat66_main_t *nm = &nat66_main;
2971   u32 translation_buckets = 1024;
2972   u32 translation_memory_size = 128 << 20;
2973   u32 user_buckets = 128;
2974   u32 user_memory_size = 64 << 20;
2975   u32 max_translations_per_user = 100;
2976   u32 outside_vrf_id = 0;
2977   u32 outside_ip6_vrf_id = 0;
2978   u32 inside_vrf_id = 0;
2979   u32 static_mapping_buckets = 1024;
2980   u32 static_mapping_memory_size = 64 << 20;
2981   u32 nat64_bib_buckets = 1024;
2982   u32 nat64_bib_memory_size = 128 << 20;
2983   u32 nat64_st_buckets = 2048;
2984   u32 nat64_st_memory_size = 256 << 20;
2985   u8 static_mapping_only = 0;
2986   u8 static_mapping_connection_tracking = 0;
2987   snat_main_per_thread_data_t *tsm;
2988   dslite_main_t *dm = &dslite_main;
2989
2990   sm->deterministic = 0;
2991   sm->out2in_dpo = 0;
2992   sm->endpoint_dependent = 0;
2993
2994   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2995     {
2996       if (unformat
2997           (input, "translation hash buckets %d", &translation_buckets))
2998         ;
2999       else if (unformat (input, "translation hash memory %d",
3000                          &translation_memory_size));
3001       else if (unformat (input, "user hash buckets %d", &user_buckets))
3002         ;
3003       else if (unformat (input, "user hash memory %d", &user_memory_size))
3004         ;
3005       else if (unformat (input, "max translations per user %d",
3006                          &max_translations_per_user))
3007         ;
3008       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3009         ;
3010       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3011         ;
3012       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3013         ;
3014       else if (unformat (input, "static mapping only"))
3015         {
3016           static_mapping_only = 1;
3017           if (unformat (input, "connection tracking"))
3018             static_mapping_connection_tracking = 1;
3019         }
3020       else if (unformat (input, "deterministic"))
3021         sm->deterministic = 1;
3022       else if (unformat (input, "nat64 bib hash buckets %d",
3023                          &nat64_bib_buckets))
3024         ;
3025       else if (unformat (input, "nat64 bib hash memory %d",
3026                          &nat64_bib_memory_size))
3027         ;
3028       else
3029         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3030         ;
3031       else if (unformat (input, "nat64 st hash memory %d",
3032                          &nat64_st_memory_size))
3033         ;
3034       else if (unformat (input, "out2in dpo"))
3035         sm->out2in_dpo = 1;
3036       else if (unformat (input, "dslite ce"))
3037         dslite_set_ce (dm, 1);
3038       else if (unformat (input, "endpoint-dependent"))
3039         sm->endpoint_dependent = 1;
3040       else
3041         return clib_error_return (0, "unknown input '%U'",
3042                                   format_unformat_error, input);
3043     }
3044
3045   if (sm->deterministic && sm->endpoint_dependent)
3046     return clib_error_return (0,
3047                               "deterministic and endpoint-dependent modes are mutually exclusive");
3048
3049   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3050     return clib_error_return (0,
3051                               "static mapping only mode available only for simple nat");
3052
3053   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3054     return clib_error_return (0,
3055                               "out2in dpo mode available only for simple nat");
3056
3057   /* for show commands, etc. */
3058   sm->translation_buckets = translation_buckets;
3059   sm->translation_memory_size = translation_memory_size;
3060   /* do not exceed load factor 10 */
3061   sm->max_translations = 10 * translation_buckets;
3062   sm->user_buckets = user_buckets;
3063   sm->user_memory_size = user_memory_size;
3064   sm->max_translations_per_user = max_translations_per_user;
3065   sm->outside_vrf_id = outside_vrf_id;
3066   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3067                                                              outside_vrf_id,
3068                                                              FIB_SOURCE_PLUGIN_HI);
3069   nm->outside_vrf_id = outside_ip6_vrf_id;
3070   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3071                                                              outside_ip6_vrf_id,
3072                                                              FIB_SOURCE_PLUGIN_HI);
3073   sm->inside_vrf_id = inside_vrf_id;
3074   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3075                                                             inside_vrf_id,
3076                                                             FIB_SOURCE_PLUGIN_HI);
3077   sm->static_mapping_only = static_mapping_only;
3078   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3079
3080   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3081                   nat64_st_memory_size);
3082
3083   if (sm->deterministic)
3084     {
3085       sm->in2out_node_index = snat_det_in2out_node.index;
3086       sm->in2out_output_node_index = ~0;
3087       sm->out2in_node_index = snat_det_out2in_node.index;
3088       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3089       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3090     }
3091   else
3092     {
3093       if (sm->endpoint_dependent)
3094         {
3095           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3096           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3097           sm->in2out_node_index = nat44_ed_in2out_node.index;
3098           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3099           sm->out2in_node_index = nat44_ed_out2in_node.index;
3100           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3101           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3102           nat_affinity_init (vm);
3103         }
3104       else
3105         {
3106           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3107           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3108           sm->in2out_node_index = snat_in2out_node.index;
3109           sm->in2out_output_node_index = snat_in2out_output_node.index;
3110           sm->out2in_node_index = snat_out2in_node.index;
3111           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3112           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3113         }
3114       if (!static_mapping_only ||
3115           (static_mapping_only && static_mapping_connection_tracking))
3116         {
3117           /* *INDENT-OFF* */
3118           vec_foreach (tsm, sm->per_thread_data)
3119             {
3120               if (sm->endpoint_dependent)
3121                 {
3122                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3123                                          translation_buckets,
3124                                          translation_memory_size);
3125                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3126                                                       format_ed_session_kvp);
3127
3128                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3129                                          translation_buckets,
3130                                          translation_memory_size);
3131                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3132                                                       format_ed_session_kvp);
3133                 }
3134               else
3135                 {
3136                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3137                                         translation_buckets,
3138                                         translation_memory_size);
3139                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3140                                                      format_session_kvp);
3141
3142                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3143                                         translation_buckets,
3144                                         translation_memory_size);
3145                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3146                                                      format_session_kvp);
3147                 }
3148
3149               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3150                                     user_memory_size);
3151               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3152                                                  format_user_kvp);
3153             }
3154           /* *INDENT-ON* */
3155
3156         }
3157       else
3158         {
3159           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3160           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3161         }
3162       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3163                             "static_mapping_by_local", static_mapping_buckets,
3164                             static_mapping_memory_size);
3165       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3166                                          format_static_mapping_kvp);
3167
3168       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3169                             "static_mapping_by_external",
3170                             static_mapping_buckets,
3171                             static_mapping_memory_size);
3172       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3173                                          format_static_mapping_kvp);
3174     }
3175
3176   return 0;
3177 }
3178
3179 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3180
3181 static void
3182 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3183                                  uword opaque,
3184                                  u32 sw_if_index,
3185                                  ip4_address_t * address,
3186                                  u32 address_length,
3187                                  u32 if_address_index, u32 is_delete)
3188 {
3189   snat_main_t *sm = &snat_main;
3190   snat_static_map_resolve_t *rp;
3191   snat_static_mapping_t *m;
3192   snat_session_key_t m_key;
3193   clib_bihash_kv_8_8_t kv, value;
3194   int i, rv;
3195   ip4_address_t l_addr;
3196
3197   for (i = 0; i < vec_len (sm->to_resolve); i++)
3198     {
3199       rp = sm->to_resolve + i;
3200       if (rp->addr_only == 0)
3201         continue;
3202       if (rp->sw_if_index == sw_if_index)
3203         goto match;
3204     }
3205
3206   return;
3207
3208 match:
3209   m_key.addr.as_u32 = address->as_u32;
3210   m_key.port = rp->addr_only ? 0 : rp->e_port;
3211   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3212   m_key.fib_index = sm->outside_fib_index;
3213   kv.key = m_key.as_u64;
3214   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3215     m = 0;
3216   else
3217     m = pool_elt_at_index (sm->static_mappings, value.value);
3218
3219   if (!is_delete)
3220     {
3221       /* Don't trip over lease renewal, static config */
3222       if (m)
3223         return;
3224     }
3225   else
3226     {
3227       if (!m)
3228         return;
3229     }
3230
3231   /* Indetity mapping? */
3232   if (rp->l_addr.as_u32 == 0)
3233     l_addr.as_u32 = address[0].as_u32;
3234   else
3235     l_addr.as_u32 = rp->l_addr.as_u32;
3236   /* Add the static mapping */
3237   rv = snat_add_static_mapping (l_addr,
3238                                 address[0],
3239                                 rp->l_port,
3240                                 rp->e_port,
3241                                 rp->vrf_id,
3242                                 rp->addr_only, ~0 /* sw_if_index */ ,
3243                                 rp->proto, !is_delete, rp->twice_nat,
3244                                 rp->out2in_only, rp->tag, rp->identity_nat);
3245   if (rv)
3246     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3247 }
3248
3249 static void
3250 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3251                                        uword opaque,
3252                                        u32 sw_if_index,
3253                                        ip4_address_t * address,
3254                                        u32 address_length,
3255                                        u32 if_address_index, u32 is_delete)
3256 {
3257   snat_main_t *sm = &snat_main;
3258   snat_static_map_resolve_t *rp;
3259   ip4_address_t l_addr;
3260   int i, j;
3261   int rv;
3262   u8 twice_nat = 0;
3263   snat_address_t *addresses = sm->addresses;
3264
3265   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3266     {
3267       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3268         goto match;
3269     }
3270
3271   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3272     {
3273       twice_nat = 1;
3274       addresses = sm->twice_nat_addresses;
3275       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3276         goto match;
3277     }
3278
3279   return;
3280
3281 match:
3282   if (!is_delete)
3283     {
3284       /* Don't trip over lease renewal, static config */
3285       for (j = 0; j < vec_len (addresses); j++)
3286         if (addresses[j].addr.as_u32 == address->as_u32)
3287           return;
3288
3289       (void) snat_add_address (sm, address, ~0, twice_nat);
3290       /* Scan static map resolution vector */
3291       for (j = 0; j < vec_len (sm->to_resolve); j++)
3292         {
3293           rp = sm->to_resolve + j;
3294           if (rp->addr_only)
3295             continue;
3296           /* On this interface? */
3297           if (rp->sw_if_index == sw_if_index)
3298             {
3299               /* Indetity mapping? */
3300               if (rp->l_addr.as_u32 == 0)
3301                 l_addr.as_u32 = address[0].as_u32;
3302               else
3303                 l_addr.as_u32 = rp->l_addr.as_u32;
3304               /* Add the static mapping */
3305               rv = snat_add_static_mapping (l_addr,
3306                                             address[0],
3307                                             rp->l_port,
3308                                             rp->e_port,
3309                                             rp->vrf_id,
3310                                             rp->addr_only,
3311                                             ~0 /* sw_if_index */ ,
3312                                             rp->proto,
3313                                             rp->is_add, rp->twice_nat,
3314                                             rp->out2in_only, rp->tag,
3315                                             rp->identity_nat);
3316               if (rv)
3317                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3318             }
3319         }
3320       return;
3321     }
3322   else
3323     {
3324       (void) snat_del_address (sm, address[0], 1, twice_nat);
3325       return;
3326     }
3327 }
3328
3329
3330 int
3331 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3332                             u8 twice_nat)
3333 {
3334   ip4_main_t *ip4_main = sm->ip4_main;
3335   ip4_address_t *first_int_addr;
3336   snat_static_map_resolve_t *rp;
3337   u32 *indices_to_delete = 0;
3338   int i, j;
3339   u32 *auto_add_sw_if_indices =
3340     twice_nat ? sm->
3341     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3342
3343   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3344     );
3345
3346   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3347     {
3348       if (auto_add_sw_if_indices[i] == sw_if_index)
3349         {
3350           if (is_del)
3351             {
3352               /* if have address remove it */
3353               if (first_int_addr)
3354                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3355               else
3356                 {
3357                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3358                     {
3359                       rp = sm->to_resolve + j;
3360                       if (rp->sw_if_index == sw_if_index)
3361                         vec_add1 (indices_to_delete, j);
3362                     }
3363                   if (vec_len (indices_to_delete))
3364                     {
3365                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3366                         vec_del1 (sm->to_resolve, j);
3367                       vec_free (indices_to_delete);
3368                     }
3369                 }
3370               if (twice_nat)
3371                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3372               else
3373                 vec_del1 (sm->auto_add_sw_if_indices, i);
3374             }
3375           else
3376             return VNET_API_ERROR_VALUE_EXIST;
3377
3378           return 0;
3379         }
3380     }
3381
3382   if (is_del)
3383     return VNET_API_ERROR_NO_SUCH_ENTRY;
3384
3385   /* add to the auto-address list */
3386   if (twice_nat)
3387     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3388   else
3389     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3390
3391   /* If the address is already bound - or static - add it now */
3392   if (first_int_addr)
3393     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3394
3395   return 0;
3396 }
3397
3398 int
3399 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3400                    snat_protocol_t proto, u32 vrf_id, int is_in)
3401 {
3402   snat_main_per_thread_data_t *tsm;
3403   clib_bihash_kv_8_8_t kv, value;
3404   ip4_header_t ip;
3405   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3406   snat_session_key_t key;
3407   snat_session_t *s;
3408   clib_bihash_8_8_t *t;
3409
3410   if (sm->endpoint_dependent)
3411     return VNET_API_ERROR_UNSUPPORTED;
3412
3413   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3414   if (sm->num_workers > 1)
3415     tsm =
3416       vec_elt_at_index (sm->per_thread_data,
3417                         sm->worker_in2out_cb (&ip, fib_index));
3418   else
3419     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3420
3421   key.addr.as_u32 = addr->as_u32;
3422   key.port = clib_host_to_net_u16 (port);
3423   key.protocol = proto;
3424   key.fib_index = fib_index;
3425   kv.key = key.as_u64;
3426   t = is_in ? &tsm->in2out : &tsm->out2in;
3427   if (!clib_bihash_search_8_8 (t, &kv, &value))
3428     {
3429       if (pool_is_free_index (tsm->sessions, value.value))
3430         return VNET_API_ERROR_UNSPECIFIED;
3431
3432       s = pool_elt_at_index (tsm->sessions, value.value);
3433       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3434       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3435       return 0;
3436     }
3437
3438   return VNET_API_ERROR_NO_SUCH_ENTRY;
3439 }
3440
3441 int
3442 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3443                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3444                       u32 vrf_id, int is_in)
3445 {
3446   ip4_header_t ip;
3447   clib_bihash_16_8_t *t;
3448   nat_ed_ses_key_t key;
3449   clib_bihash_kv_16_8_t kv, value;
3450   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3451   snat_session_t *s;
3452   snat_main_per_thread_data_t *tsm;
3453
3454   if (!sm->endpoint_dependent)
3455     return VNET_API_ERROR_FEATURE_DISABLED;
3456
3457   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3458   if (sm->num_workers > 1)
3459     tsm =
3460       vec_elt_at_index (sm->per_thread_data,
3461                         sm->worker_in2out_cb (&ip, fib_index));
3462   else
3463     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3464
3465   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3466   key.l_addr.as_u32 = addr->as_u32;
3467   key.r_addr.as_u32 = eh_addr->as_u32;
3468   key.l_port = clib_host_to_net_u16 (port);
3469   key.r_port = clib_host_to_net_u16 (eh_port);
3470   key.proto = proto;
3471   key.fib_index = fib_index;
3472   kv.key[0] = key.as_u64[0];
3473   kv.key[1] = key.as_u64[1];
3474   if (clib_bihash_search_16_8 (t, &kv, &value))
3475     return VNET_API_ERROR_NO_SUCH_ENTRY;
3476
3477   if (pool_is_free_index (tsm->sessions, value.value))
3478     return VNET_API_ERROR_UNSPECIFIED;
3479   s = pool_elt_at_index (tsm->sessions, value.value);
3480   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3481   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3482   return 0;
3483 }
3484
3485 void
3486 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3487 {
3488   snat_main_t *sm = &snat_main;
3489
3490   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3491   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3492   sm->psid = psid;
3493   sm->psid_offset = psid_offset;
3494   sm->psid_length = psid_length;
3495 }
3496
3497 void
3498 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3499 {
3500   snat_main_t *sm = &snat_main;
3501
3502   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3503   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3504   sm->start_port = start_port;
3505   sm->end_port = end_port;
3506 }
3507
3508 void
3509 nat_set_alloc_addr_and_port_default (void)
3510 {
3511   snat_main_t *sm = &snat_main;
3512
3513   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3514   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3515 }
3516
3517 /*
3518  * fd.io coding-style-patch-verification: ON
3519  *
3520  * Local Variables:
3521  * eval: (c-set-style "gnu")
3522  * End:
3523  */