src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35
  36 #include <vpp/app/version.h>
  37
  38 snat_main_t snat_main;
  39
  40 /* *INDENT-OFF* */
  41
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat44-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  47 };
  48 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat44-out2in",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-dhcp-client-detect"),
  53 };
  54 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-classify",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-in2out",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-out2in",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-det-classify",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  74 };
  75 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-ed-in2out",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-ed-out2in",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  84                                "ip4-dhcp-client-detect"),
  85 };
  86 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-ed-classify",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-in2out-worker-handoff",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  95 };
  96 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-out2in-worker-handoff",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-handoff-classify",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-in2out-fast",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 111 };
 112 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-out2in-fast",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 116                                "ip4-dhcp-client-detect"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-hairpin-dst",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ed-hairpin-dst",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 127 };
 128
 129 /* Hook up output features */
 130 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 131   .arc_name = "ip4-output",
 132   .node_name = "nat44-in2out-output",
 133   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 136   .arc_name = "ip4-output",
 137   .node_name = "nat44-in2out-output-worker-handoff",
 138   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 141   .arc_name = "ip4-output",
 142   .node_name = "nat44-hairpin-src",
 143   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 144 };
 145 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-ed-in2out-output",
 148   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 149 };
 150 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-ed-hairpin-src",
 153   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 154 };
 155
 156 /* Hook up ip4-local features */
 157 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 158 {
 159   .arc_name = "ip4-local",
 160   .node_name = "nat44-hairpinning",
 161   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 164 {
 165   .arc_name = "ip4-local",
 166   .node_name = "nat44-ed-hairpinning",
 167   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 168 };
 169
 170
 171 VLIB_PLUGIN_REGISTER () = {
 172     .version = VPP_BUILD_VER,
 173     .description = "Network Address Translation",
 174 };
 175 /* *INDENT-ON* */
 176
 177 void
 178 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 179 {
 180   snat_session_key_t key;
 181   clib_bihash_kv_8_8_t kv;
 182   nat_ed_ses_key_t ed_key;
 183   clib_bihash_kv_16_8_t ed_kv;
 184   snat_main_per_thread_data_t *tsm =
 185     vec_elt_at_index (sm->per_thread_data, thread_index);
 186
 187   if (is_fwd_bypass_session (s))
 188     {
 189       ed_key.l_addr = s->in2out.addr;
 190       ed_key.r_addr = s->ext_host_addr;
 191       ed_key.l_port = s->in2out.port;
 192       ed_key.r_port = s->ext_host_port;
 193       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 194       ed_key.fib_index = 0;
 195       ed_kv.key[0] = ed_key.as_u64[0];
 196       ed_kv.key[1] = ed_key.as_u64[1];
 197       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 198         nat_log_warn ("in2out_ed key del failed");
 199       return;
 200     }
 201
 202   /* session lookup tables */
 203   if (is_ed_session (s))
 204     {
 205       if (is_affinity_sessions (s))
 206         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 207                              s->in2out.protocol, s->out2in.port);
 208       ed_key.l_addr = s->out2in.addr;
 209       ed_key.r_addr = s->ext_host_addr;
 210       ed_key.fib_index = s->out2in.fib_index;
 211       if (snat_is_unk_proto_session (s))
 212         {
 213           ed_key.proto = s->in2out.port;
 214           ed_key.r_port = 0;
 215           ed_key.l_port = 0;
 216         }
 217       else
 218         {
 219           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 220           ed_key.l_port = s->out2in.port;
 221           ed_key.r_port = s->ext_host_port;
 222         }
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 226         nat_log_warn ("out2in_ed key del failed");
 227       ed_key.l_addr = s->in2out.addr;
 228       ed_key.fib_index = s->in2out.fib_index;
 229       if (!snat_is_unk_proto_session (s))
 230         ed_key.l_port = s->in2out.port;
 231       if (is_twice_nat_session (s))
 232         {
 233           ed_key.r_addr = s->ext_host_nat_addr;
 234           ed_key.r_port = s->ext_host_nat_port;
 235         }
 236       ed_kv.key[0] = ed_key.as_u64[0];
 237       ed_kv.key[1] = ed_key.as_u64[1];
 238       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 239         nat_log_warn ("in2out_ed key del failed");
 240
 241       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 242                              &s->in2out.addr, s->in2out.port,
 243                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 244                              &s->out2in.addr, s->out2in.port,
 245                              &s->ext_host_addr, s->ext_host_port,
 246                              s->in2out.protocol, is_twice_nat_session (s));
 247     }
 248   else
 249     {
 250       kv.key = s->in2out.as_u64;
 251       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 252         nat_log_warn ("in2out key del failed");
 253       kv.key = s->out2in.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 255         nat_log_warn ("out2in key del failed");
 256
 257       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 258                                &s->in2out.addr, s->in2out.port,
 259                                &s->out2in.addr, s->out2in.port,
 260                                s->in2out.protocol);
 261     }
 262
 263   if (snat_is_unk_proto_session (s))
 264     return;
 265
 266   /* log NAT event */
 267   snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 268                                        s->out2in.addr.as_u32,
 269                                        s->in2out.protocol,
 270                                        s->in2out.port,
 271                                        s->out2in.port, s->in2out.fib_index);
 272
 273   /* Twice NAT address and port for external host */
 274   if (is_twice_nat_session (s))
 275     {
 276       key.protocol = s->in2out.protocol;
 277       key.port = s->ext_host_nat_port;
 278       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 279       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 280                                           thread_index, &key);
 281     }
 282
 283   if (snat_is_session_static (s))
 284     return;
 285
 286   snat_free_outside_address_and_port (sm->addresses, thread_index,
 287                                       &s->out2in);
 288 }
 289
 290 snat_user_t *
 291 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 292                         u32 thread_index)
 293 {
 294   snat_user_t *u = 0;
 295   snat_user_key_t user_key;
 296   clib_bihash_kv_8_8_t kv, value;
 297   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 298   dlist_elt_t *per_user_list_head_elt;
 299
 300   user_key.addr.as_u32 = addr->as_u32;
 301   user_key.fib_index = fib_index;
 302   kv.key = user_key.as_u64;
 303
 304   /* Ever heard of the "user" = src ip4 address before? */
 305   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 306     {
 307       /* no, make a new one */
 308       pool_get (tsm->users, u);
 309       clib_memset (u, 0, sizeof (*u));
 310       u->addr.as_u32 = addr->as_u32;
 311       u->fib_index = fib_index;
 312
 313       pool_get (tsm->list_pool, per_user_list_head_elt);
 314
 315       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 316         tsm->list_pool;
 317
 318       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 319
 320       kv.value = u - tsm->users;
 321
 322       /* add user */
 323       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 324         nat_log_warn ("user_hash keay add failed");
 325
 326       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 327                                pool_elts (tsm->users));
 328     }
 329   else
 330     {
 331       u = pool_elt_at_index (tsm->users, value.value);
 332     }
 333
 334   return u;
 335 }
 336
 337 snat_session_t *
 338 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 339                               u32 thread_index)
 340 {
 341   snat_session_t *s;
 342   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 343   u32 oldest_per_user_translation_list_index, session_index;
 344   dlist_elt_t *oldest_per_user_translation_list_elt;
 345   dlist_elt_t *per_user_translation_list_elt;
 346
 347   /* Over quota? Recycle the least recently used translation */
 348   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 349     {
 350       oldest_per_user_translation_list_index =
 351         clib_dlist_remove_head (tsm->list_pool,
 352                                 u->sessions_per_user_list_head_index);
 353
 354       ASSERT (oldest_per_user_translation_list_index != ~0);
 355
 356       /* Add it back to the end of the LRU list */
 357       clib_dlist_addtail (tsm->list_pool,
 358                           u->sessions_per_user_list_head_index,
 359                           oldest_per_user_translation_list_index);
 360       /* Get the list element */
 361       oldest_per_user_translation_list_elt =
 362         pool_elt_at_index (tsm->list_pool,
 363                            oldest_per_user_translation_list_index);
 364
 365       /* Get the session index from the list element */
 366       session_index = oldest_per_user_translation_list_elt->value;
 367
 368       /* Get the session */
 369       s = pool_elt_at_index (tsm->sessions, session_index);
 370       nat_free_session_data (sm, s, thread_index);
 371       if (snat_is_session_static (s))
 372         u->nstaticsessions--;
 373       else
 374         u->nsessions--;
 375       s->flags = 0;
 376       s->total_bytes = 0;
 377       s->total_pkts = 0;
 378       s->state = 0;
 379       s->ext_host_addr.as_u32 = 0;
 380       s->ext_host_port = 0;
 381       s->ext_host_nat_addr.as_u32 = 0;
 382       s->ext_host_nat_port = 0;
 383     }
 384   else
 385     {
 386       pool_get (tsm->sessions, s);
 387       clib_memset (s, 0, sizeof (*s));
 388
 389       /* Create list elts */
 390       pool_get (tsm->list_pool, per_user_translation_list_elt);
 391       clib_dlist_init (tsm->list_pool,
 392                        per_user_translation_list_elt - tsm->list_pool);
 393
 394       per_user_translation_list_elt->value = s - tsm->sessions;
 395       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 396       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 397
 398       clib_dlist_addtail (tsm->list_pool,
 399                           s->per_user_list_head_index,
 400                           per_user_translation_list_elt - tsm->list_pool);
 401
 402       s->user_index = u - tsm->users;
 403       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 404                                pool_elts (tsm->sessions));
 405     }
 406
 407   return s;
 408 }
 409
 410 snat_session_t *
 411 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 412                       f64 now)
 413 {
 414   snat_session_t *s;
 415   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 416   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 417   u32 oldest_index;
 418   u64 sess_timeout_time;
 419
 420   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 421     goto alloc_new;
 422
 423   oldest_index =
 424     clib_dlist_remove_head (tsm->list_pool,
 425                             u->sessions_per_user_list_head_index);
 426   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 427   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 428   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 429   if (now >= sess_timeout_time)
 430     {
 431       clib_dlist_addtail (tsm->list_pool,
 432                           u->sessions_per_user_list_head_index, oldest_index);
 433       nat_free_session_data (sm, s, thread_index);
 434       if (snat_is_session_static (s))
 435         u->nstaticsessions--;
 436       else
 437         u->nsessions--;
 438       s->flags = 0;
 439       s->total_bytes = 0;
 440       s->total_pkts = 0;
 441       s->state = 0;
 442       s->ext_host_addr.as_u32 = 0;
 443       s->ext_host_port = 0;
 444       s->ext_host_nat_addr.as_u32 = 0;
 445       s->ext_host_nat_port = 0;
 446     }
 447   else
 448     {
 449       clib_dlist_addhead (tsm->list_pool,
 450                           u->sessions_per_user_list_head_index, oldest_index);
 451       if ((u->nsessions + u->nstaticsessions) >=
 452           sm->max_translations_per_user)
 453         {
 454           nat_log_warn ("max translations per user %U", format_ip4_address,
 455                         &u->addr);
 456           snat_ipfix_logging_max_entries_per_user
 457             (sm->max_translations_per_user, u->addr.as_u32);
 458           return 0;
 459         }
 460       else
 461         {
 462         alloc_new:
 463           pool_get (tsm->sessions, s);
 464           clib_memset (s, 0, sizeof (*s));
 465
 466           /* Create list elts */
 467           pool_get (tsm->list_pool, per_user_translation_list_elt);
 468           clib_dlist_init (tsm->list_pool,
 469                            per_user_translation_list_elt - tsm->list_pool);
 470
 471           per_user_translation_list_elt->value = s - tsm->sessions;
 472           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 473           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 474
 475           clib_dlist_addtail (tsm->list_pool,
 476                               s->per_user_list_head_index,
 477                               per_user_translation_list_elt - tsm->list_pool);
 478         }
 479
 480       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 481                                pool_elts (tsm->sessions));
 482     }
 483
 484   return s;
 485 }
 486
 487 void
 488 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 489                           int is_add)
 490 {
 491   fib_prefix_t prefix = {
 492     .fp_len = p_len,
 493     .fp_proto = FIB_PROTOCOL_IP4,
 494     .fp_addr = {
 495                 .ip4.as_u32 = addr->as_u32,
 496                 },
 497   };
 498   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 499
 500   if (is_add)
 501     fib_table_entry_update_one_path (fib_index,
 502                                      &prefix,
 503                                      FIB_SOURCE_PLUGIN_LOW,
 504                                      (FIB_ENTRY_FLAG_CONNECTED |
 505                                       FIB_ENTRY_FLAG_LOCAL |
 506                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 507                                      DPO_PROTO_IP4,
 508                                      NULL,
 509                                      sw_if_index,
 510                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 511   else
 512     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 513 }
 514
 515 int
 516 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 517                   u8 twice_nat)
 518 {
 519   snat_address_t *ap;
 520   snat_interface_t *i;
 521   vlib_thread_main_t *tm = vlib_get_thread_main ();
 522
 523   if (twice_nat && !sm->endpoint_dependent)
 524     return VNET_API_ERROR_FEATURE_DISABLED;
 525
 526   /* Check if address already exists */
 527   /* *INDENT-OFF* */
 528   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 529     {
 530       if (ap->addr.as_u32 == addr->as_u32)
 531         return VNET_API_ERROR_VALUE_EXIST;
 532     }
 533   /* *INDENT-ON* */
 534
 535   if (twice_nat)
 536     vec_add2 (sm->twice_nat_addresses, ap, 1);
 537   else
 538     vec_add2 (sm->addresses, ap, 1);
 539
 540   ap->addr = *addr;
 541   if (vrf_id != ~0)
 542     ap->fib_index =
 543       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 544                                          FIB_SOURCE_PLUGIN_LOW);
 545   else
 546     ap->fib_index = ~0;
 547 #define _(N, i, n, s) \
 548   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 549   ap->busy_##n##_ports = 0; \
 550   ap->busy_##n##_ports_per_thread = 0;\
 551   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 552   foreach_snat_protocol
 553 #undef _
 554     if (twice_nat)
 555     return 0;
 556
 557   /* Add external address to FIB */
 558   /* *INDENT-OFF* */
 559   pool_foreach (i, sm->interfaces,
 560   ({
 561     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 562       continue;
 563
 564     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 565     break;
 566   }));
 567   pool_foreach (i, sm->output_feature_interfaces,
 568   ({
 569     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 570       continue;
 571
 572     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 573     break;
 574   }));
 575   /* *INDENT-ON* */
 576
 577   return 0;
 578 }
 579
 580 static int
 581 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 582 {
 583   snat_static_mapping_t *m;
 584   /* *INDENT-OFF* */
 585   pool_foreach (m, sm->static_mappings,
 586   ({
 587       if (is_addr_only_static_mapping (m) ||
 588           is_out2in_only_static_mapping (m) ||
 589           is_identity_static_mapping (m))
 590         continue;
 591       if (m->external_addr.as_u32 == addr.as_u32)
 592         return 1;
 593   }));
 594   /* *INDENT-ON* */
 595
 596   return 0;
 597 }
 598
 599 void
 600 increment_v4_address (ip4_address_t * a)
 601 {
 602   u32 v;
 603
 604   v = clib_net_to_host_u32 (a->as_u32) + 1;
 605   a->as_u32 = clib_host_to_net_u32 (v);
 606 }
 607
 608 static void
 609 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 610                                        ip4_address_t l_addr,
 611                                        u16 l_port,
 612                                        u32 sw_if_index,
 613                                        u16 e_port,
 614                                        u32 vrf_id,
 615                                        snat_protocol_t proto,
 616                                        int addr_only, int is_add, u8 * tag,
 617                                        int twice_nat, int out2in_only,
 618                                        int identity_nat)
 619 {
 620   snat_static_map_resolve_t *rp;
 621
 622   vec_add2 (sm->to_resolve, rp, 1);
 623   rp->l_addr.as_u32 = l_addr.as_u32;
 624   rp->l_port = l_port;
 625   rp->sw_if_index = sw_if_index;
 626   rp->e_port = e_port;
 627   rp->vrf_id = vrf_id;
 628   rp->proto = proto;
 629   rp->addr_only = addr_only;
 630   rp->is_add = is_add;
 631   rp->twice_nat = twice_nat;
 632   rp->out2in_only = out2in_only;
 633   rp->identity_nat = identity_nat;
 634   rp->tag = vec_dup (tag);
 635 }
 636
 637 static u32
 638 get_thread_idx_by_port (u16 e_port)
 639 {
 640   snat_main_t *sm = &snat_main;
 641   u32 thread_idx = sm->num_workers;
 642   if (sm->num_workers > 1)
 643     {
 644       thread_idx =
 645         sm->first_worker_index +
 646         sm->workers[(e_port - 1024) / sm->port_per_thread];
 647     }
 648   return thread_idx;
 649 }
 650
 651 int
 652 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 653                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 654                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 655                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 656                          u8 identity_nat)
 657 {
 658   snat_main_t *sm = &snat_main;
 659   snat_static_mapping_t *m;
 660   snat_session_key_t m_key;
 661   clib_bihash_kv_8_8_t kv, value;
 662   snat_address_t *a = 0;
 663   u32 fib_index = ~0;
 664   snat_interface_t *interface;
 665   int i;
 666   snat_main_per_thread_data_t *tsm;
 667   snat_user_key_t u_key;
 668   snat_user_t *u;
 669   dlist_elt_t *head, *elt;
 670   u32 elt_index, head_index;
 671   u32 ses_index;
 672   u64 user_index;
 673   snat_session_t *s;
 674   snat_static_map_resolve_t *rp, *rp_match = 0;
 675   nat44_lb_addr_port_t *local;
 676   u32 find = ~0;
 677
 678   if (!sm->endpoint_dependent)
 679     {
 680       if (twice_nat || out2in_only)
 681         return VNET_API_ERROR_FEATURE_DISABLED;
 682     }
 683
 684   /* If the external address is a specific interface address */
 685   if (sw_if_index != ~0)
 686     {
 687       ip4_address_t *first_int_addr;
 688
 689       for (i = 0; i < vec_len (sm->to_resolve); i++)
 690         {
 691           rp = sm->to_resolve + i;
 692           if (rp->sw_if_index != sw_if_index ||
 693               rp->l_addr.as_u32 != l_addr.as_u32 ||
 694               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 695             continue;
 696
 697           if (!addr_only)
 698             {
 699               if (rp->l_port != l_port || rp->e_port != e_port
 700                   || rp->proto != proto)
 701                 continue;
 702             }
 703
 704           rp_match = rp;
 705           break;
 706         }
 707
 708       /* Might be already set... */
 709       first_int_addr = ip4_interface_first_address
 710         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 711
 712       if (is_add)
 713         {
 714           if (rp_match)
 715             return VNET_API_ERROR_VALUE_EXIST;
 716
 717           snat_add_static_mapping_when_resolved
 718             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 719              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 720
 721           /* DHCP resolution required? */
 722           if (first_int_addr == 0)
 723             {
 724               return 0;
 725             }
 726           else
 727             {
 728               e_addr.as_u32 = first_int_addr->as_u32;
 729               /* Identity mapping? */
 730               if (l_addr.as_u32 == 0)
 731                 l_addr.as_u32 = e_addr.as_u32;
 732             }
 733         }
 734       else
 735         {
 736           if (!rp_match)
 737             return VNET_API_ERROR_NO_SUCH_ENTRY;
 738
 739           vec_del1 (sm->to_resolve, i);
 740
 741           if (first_int_addr)
 742             {
 743               e_addr.as_u32 = first_int_addr->as_u32;
 744               /* Identity mapping? */
 745               if (l_addr.as_u32 == 0)
 746                 l_addr.as_u32 = e_addr.as_u32;
 747             }
 748           else
 749             return 0;
 750         }
 751     }
 752
 753   m_key.addr = e_addr;
 754   m_key.port = addr_only ? 0 : e_port;
 755   m_key.protocol = addr_only ? 0 : proto;
 756   m_key.fib_index = 0;
 757   kv.key = m_key.as_u64;
 758   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 759     m = 0;
 760   else
 761     m = pool_elt_at_index (sm->static_mappings, value.value);
 762
 763   if (is_add)
 764     {
 765       if (m)
 766         {
 767           if (is_identity_static_mapping (m))
 768             {
 769               /* *INDENT-OFF* */
 770               pool_foreach (local, m->locals,
 771               ({
 772                 if (local->vrf_id == vrf_id)
 773                   return VNET_API_ERROR_VALUE_EXIST;
 774               }));
 775               /* *INDENT-ON* */
 776               pool_get (m->locals, local);
 777               local->vrf_id = vrf_id;
 778               local->fib_index =
 779                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 780                                                    FIB_SOURCE_PLUGIN_LOW);
 781               m_key.addr = m->local_addr;
 782               m_key.port = m->local_port;
 783               m_key.protocol = m->proto;
 784               m_key.fib_index = local->fib_index;
 785               kv.key = m_key.as_u64;
 786               kv.value = m - sm->static_mappings;
 787               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 788               return 0;
 789             }
 790           else
 791             return VNET_API_ERROR_VALUE_EXIST;
 792         }
 793
 794       if (twice_nat && addr_only)
 795         return VNET_API_ERROR_UNSUPPORTED;
 796
 797       /* Convert VRF id to FIB index */
 798       if (vrf_id != ~0)
 799         fib_index =
 800           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 801                                              FIB_SOURCE_PLUGIN_LOW);
 802       /* If not specified use inside VRF id from SNAT plugin startup config */
 803       else
 804         {
 805           fib_index = sm->inside_fib_index;
 806           vrf_id = sm->inside_vrf_id;
 807         }
 808
 809       if (!(out2in_only || identity_nat))
 810         {
 811           m_key.addr = l_addr;
 812           m_key.port = addr_only ? 0 : l_port;
 813           m_key.protocol = addr_only ? 0 : proto;
 814           m_key.fib_index = fib_index;
 815           kv.key = m_key.as_u64;
 816           if (!clib_bihash_search_8_8
 817               (&sm->static_mapping_by_local, &kv, &value))
 818             return VNET_API_ERROR_VALUE_EXIST;
 819         }
 820
 821       /* Find external address in allocated addresses and reserve port for
 822          address and port pair mapping when dynamic translations enabled */
 823       if (!(addr_only || sm->static_mapping_only || out2in_only))
 824         {
 825           for (i = 0; i < vec_len (sm->addresses); i++)
 826             {
 827               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 828                 {
 829                   a = sm->addresses + i;
 830                   /* External port must be unused */
 831                   switch (proto)
 832                     {
 833 #define _(N, j, n, s) \
 834                     case SNAT_PROTOCOL_##N: \
 835                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 836                         return VNET_API_ERROR_INVALID_VALUE; \
 837                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 838                       if (e_port > 1024) \
 839                         { \
 840                           a->busy_##n##_ports++; \
 841                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 842                         } \
 843                       break;
 844                       foreach_snat_protocol
 845 #undef _
 846                     default:
 847                       nat_log_info ("unknown protocol");
 848                       return VNET_API_ERROR_INVALID_VALUE_2;
 849                     }
 850                   break;
 851                 }
 852             }
 853           /* External address must be allocated */
 854           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 855             {
 856               if (sw_if_index != ~0)
 857                 {
 858                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 859                     {
 860                       rp = sm->to_resolve + i;
 861                       if (rp->addr_only)
 862                         continue;
 863                       if (rp->sw_if_index != sw_if_index &&
 864                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 865                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 866                           rp->e_port != e_port && rp->proto != proto)
 867                         continue;
 868
 869                       vec_del1 (sm->to_resolve, i);
 870                       break;
 871                     }
 872                 }
 873               return VNET_API_ERROR_NO_SUCH_ENTRY;
 874             }
 875         }
 876
 877       pool_get (sm->static_mappings, m);
 878       clib_memset (m, 0, sizeof (*m));
 879       m->tag = vec_dup (tag);
 880       m->local_addr = l_addr;
 881       m->external_addr = e_addr;
 882       m->twice_nat = twice_nat;
 883       if (out2in_only)
 884         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 885       if (addr_only)
 886         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 887       if (identity_nat)
 888         {
 889           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 890           pool_get (m->locals, local);
 891           local->vrf_id = vrf_id;
 892           local->fib_index = fib_index;
 893         }
 894       else
 895         {
 896           m->vrf_id = vrf_id;
 897           m->fib_index = fib_index;
 898         }
 899       if (!addr_only)
 900         {
 901           m->local_port = l_port;
 902           m->external_port = e_port;
 903           m->proto = proto;
 904         }
 905
 906       if (sm->num_workers > 1)
 907         {
 908           ip4_header_t ip = {
 909             .src_address = m->local_addr,
 910           };
 911           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 912           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 913         }
 914       else
 915         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 916
 917       m_key.addr = m->local_addr;
 918       m_key.port = m->local_port;
 919       m_key.protocol = m->proto;
 920       m_key.fib_index = fib_index;
 921       kv.key = m_key.as_u64;
 922       kv.value = m - sm->static_mappings;
 923       if (!out2in_only)
 924         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 925
 926       m_key.addr = m->external_addr;
 927       m_key.port = m->external_port;
 928       m_key.fib_index = 0;
 929       kv.key = m_key.as_u64;
 930       kv.value = m - sm->static_mappings;
 931       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 932
 933       /* Delete dynamic sessions matching local address (+ local port) */
 934       if (!(sm->static_mapping_only))
 935         {
 936           u_key.addr = m->local_addr;
 937           u_key.fib_index = m->fib_index;
 938           kv.key = u_key.as_u64;
 939           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 940             {
 941               user_index = value.value;
 942               u = pool_elt_at_index (tsm->users, user_index);
 943               if (u->nsessions)
 944                 {
 945                   head_index = u->sessions_per_user_list_head_index;
 946                   head = pool_elt_at_index (tsm->list_pool, head_index);
 947                   elt_index = head->next;
 948                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 949                   ses_index = elt->value;
 950                   while (ses_index != ~0)
 951                     {
 952                       s = pool_elt_at_index (tsm->sessions, ses_index);
 953                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 954                       ses_index = elt->value;
 955
 956                       if (snat_is_session_static (s))
 957                         continue;
 958
 959                       if (!addr_only
 960                           && (clib_net_to_host_u16 (s->in2out.port) !=
 961                               m->local_port))
 962                         continue;
 963
 964                       nat_free_session_data (sm, s,
 965                                              tsm - sm->per_thread_data);
 966                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 967
 968                       if (!addr_only && !sm->endpoint_dependent)
 969                         break;
 970                     }
 971                 }
 972             }
 973         }
 974     }
 975   else
 976     {
 977       if (!m)
 978         {
 979           if (sw_if_index != ~0)
 980             return 0;
 981           else
 982             return VNET_API_ERROR_NO_SUCH_ENTRY;
 983         }
 984
 985       if (identity_nat)
 986         {
 987           if (vrf_id == ~0)
 988             vrf_id = sm->inside_vrf_id;
 989
 990           /* *INDENT-OFF* */
 991           pool_foreach (local, m->locals,
 992           ({
 993             if (local->vrf_id == vrf_id)
 994               find = local - m->locals;
 995           }));
 996           /* *INDENT-ON* */
 997           if (find == ~0)
 998             return VNET_API_ERROR_NO_SUCH_ENTRY;
 999
1000           local = pool_elt_at_index (m->locals, find);
1001           fib_index = local->fib_index;
1002           pool_put (m->locals, local);
1003         }
1004       else
1005         fib_index = m->fib_index;
1006
1007       /* Free external address port */
1008       if (!(addr_only || sm->static_mapping_only || out2in_only))
1009         {
1010           for (i = 0; i < vec_len (sm->addresses); i++)
1011             {
1012               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1013                 {
1014                   a = sm->addresses + i;
1015                   switch (proto)
1016                     {
1017 #define _(N, j, n, s) \
1018                     case SNAT_PROTOCOL_##N: \
1019                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1020                       if (e_port > 1024) \
1021                         { \
1022                           a->busy_##n##_ports--; \
1023                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1024                         } \
1025                       break;
1026                       foreach_snat_protocol
1027 #undef _
1028                     default:
1029                       nat_log_info ("unknown protocol");
1030                       return VNET_API_ERROR_INVALID_VALUE_2;
1031                     }
1032                   break;
1033                 }
1034             }
1035         }
1036
1037       if (sm->num_workers > 1)
1038         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1039       else
1040         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1041
1042       m_key.addr = m->local_addr;
1043       m_key.port = m->local_port;
1044       m_key.protocol = m->proto;
1045       m_key.fib_index = fib_index;
1046       kv.key = m_key.as_u64;
1047       if (!out2in_only)
1048         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1049
1050       /* Delete session(s) for static mapping if exist */
1051       if (!(sm->static_mapping_only) ||
1052           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1053         {
1054           u_key.addr = m->local_addr;
1055           u_key.fib_index = fib_index;
1056           kv.key = u_key.as_u64;
1057           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1058             {
1059               user_index = value.value;
1060               u = pool_elt_at_index (tsm->users, user_index);
1061               if (u->nstaticsessions)
1062                 {
1063                   head_index = u->sessions_per_user_list_head_index;
1064                   head = pool_elt_at_index (tsm->list_pool, head_index);
1065                   elt_index = head->next;
1066                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1067                   ses_index = elt->value;
1068                   while (ses_index != ~0)
1069                     {
1070                       s = pool_elt_at_index (tsm->sessions, ses_index);
1071                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1072                       ses_index = elt->value;
1073
1074                       if (!addr_only)
1075                         {
1076                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1077                               (clib_net_to_host_u16 (s->out2in.port) !=
1078                                e_port))
1079                             continue;
1080                         }
1081
1082                       if (is_lb_session (s))
1083                         continue;
1084
1085                       if (!snat_is_session_static (s))
1086                         continue;
1087
1088                       nat_free_session_data (sm, s,
1089                                              tsm - sm->per_thread_data);
1090                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1091
1092                       if (!addr_only && !sm->endpoint_dependent)
1093                         break;
1094                     }
1095                 }
1096             }
1097         }
1098
1099       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1100       if (pool_elts (m->locals))
1101         return 0;
1102
1103       m_key.addr = m->external_addr;
1104       m_key.port = m->external_port;
1105       m_key.fib_index = 0;
1106       kv.key = m_key.as_u64;
1107       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1108
1109       vec_free (m->tag);
1110       vec_free (m->workers);
1111       /* Delete static mapping from pool */
1112       pool_put (sm->static_mappings, m);
1113     }
1114
1115   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1116     return 0;
1117
1118   /* Add/delete external address to FIB */
1119   /* *INDENT-OFF* */
1120   pool_foreach (interface, sm->interfaces,
1121   ({
1122     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1123       continue;
1124
1125     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1126     break;
1127   }));
1128   pool_foreach (interface, sm->output_feature_interfaces,
1129   ({
1130     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1131       continue;
1132
1133     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1134     break;
1135   }));
1136   /* *INDENT-ON* */
1137
1138   return 0;
1139 }
1140
1141 int
1142 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1143                                  snat_protocol_t proto,
1144                                  nat44_lb_addr_port_t * locals, u8 is_add,
1145                                  twice_nat_type_t twice_nat, u8 out2in_only,
1146                                  u8 * tag, u32 affinity)
1147 {
1148   snat_main_t *sm = &snat_main;
1149   snat_static_mapping_t *m;
1150   snat_session_key_t m_key;
1151   clib_bihash_kv_8_8_t kv, value;
1152   snat_address_t *a = 0;
1153   int i;
1154   nat44_lb_addr_port_t *local;
1155   u32 elt_index, head_index, ses_index;
1156   snat_main_per_thread_data_t *tsm;
1157   snat_user_key_t u_key;
1158   snat_user_t *u;
1159   snat_session_t *s;
1160   dlist_elt_t *head, *elt;
1161   uword *bitmap = 0;
1162
1163   if (!sm->endpoint_dependent)
1164     return VNET_API_ERROR_FEATURE_DISABLED;
1165
1166   m_key.addr = e_addr;
1167   m_key.port = e_port;
1168   m_key.protocol = proto;
1169   m_key.fib_index = 0;
1170   kv.key = m_key.as_u64;
1171   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1172     m = 0;
1173   else
1174     m = pool_elt_at_index (sm->static_mappings, value.value);
1175
1176   if (is_add)
1177     {
1178       if (m)
1179         return VNET_API_ERROR_VALUE_EXIST;
1180
1181       if (vec_len (locals) < 2)
1182         return VNET_API_ERROR_INVALID_VALUE;
1183
1184       /* Find external address in allocated addresses and reserve port for
1185          address and port pair mapping when dynamic translations enabled */
1186       if (!(sm->static_mapping_only || out2in_only))
1187         {
1188           for (i = 0; i < vec_len (sm->addresses); i++)
1189             {
1190               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1191                 {
1192                   a = sm->addresses + i;
1193                   /* External port must be unused */
1194                   switch (proto)
1195                     {
1196 #define _(N, j, n, s) \
1197                     case SNAT_PROTOCOL_##N: \
1198                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1199                         return VNET_API_ERROR_INVALID_VALUE; \
1200                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1201                       if (e_port > 1024) \
1202                         { \
1203                           a->busy_##n##_ports++; \
1204                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1205                         } \
1206                       break;
1207                       foreach_snat_protocol
1208 #undef _
1209                     default:
1210                       nat_log_info ("unknown protocol");
1211                       return VNET_API_ERROR_INVALID_VALUE_2;
1212                     }
1213                   break;
1214                 }
1215             }
1216           /* External address must be allocated */
1217           if (!a)
1218             return VNET_API_ERROR_NO_SUCH_ENTRY;
1219         }
1220
1221       pool_get (sm->static_mappings, m);
1222       clib_memset (m, 0, sizeof (*m));
1223       m->tag = vec_dup (tag);
1224       m->external_addr = e_addr;
1225       m->external_port = e_port;
1226       m->proto = proto;
1227       m->twice_nat = twice_nat;
1228       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1229       if (out2in_only)
1230         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1231       m->affinity = affinity;
1232
1233       if (affinity)
1234         m->affinity_per_service_list_head_index =
1235           nat_affinity_get_per_service_list_head_index ();
1236       else
1237         m->affinity_per_service_list_head_index = ~0;
1238
1239       m_key.addr = m->external_addr;
1240       m_key.port = m->external_port;
1241       m_key.protocol = m->proto;
1242       m_key.fib_index = 0;
1243       kv.key = m_key.as_u64;
1244       kv.value = m - sm->static_mappings;
1245       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1246         {
1247           nat_log_err ("static_mapping_by_external key add failed");
1248           return VNET_API_ERROR_UNSPECIFIED;
1249         }
1250
1251       m_key.fib_index = m->fib_index;
1252       for (i = 0; i < vec_len (locals); i++)
1253         {
1254           locals[i].fib_index =
1255             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1256                                                locals[i].vrf_id,
1257                                                FIB_SOURCE_PLUGIN_LOW);
1258           m_key.addr = locals[i].addr;
1259           m_key.fib_index = locals[i].fib_index;
1260           if (!out2in_only)
1261             {
1262               m_key.port = locals[i].port;
1263               kv.key = m_key.as_u64;
1264               kv.value = m - sm->static_mappings;
1265               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1266             }
1267           locals[i].prefix = (i == 0) ? locals[i].probability :
1268             (locals[i - 1].prefix + locals[i].probability);
1269           pool_get (m->locals, local);
1270           *local = locals[i];
1271           if (sm->num_workers > 1)
1272             {
1273               ip4_header_t ip = {
1274                 .src_address = locals[i].addr,
1275               };
1276               bitmap =
1277                 clib_bitmap_set (bitmap,
1278                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1279             }
1280         }
1281
1282       /* Assign workers */
1283       if (sm->num_workers > 1)
1284         {
1285           /* *INDENT-OFF* */
1286           clib_bitmap_foreach (i, bitmap,
1287             ({
1288                vec_add1(m->workers, i);
1289             }));
1290           /* *INDENT-ON* */
1291         }
1292     }
1293   else
1294     {
1295       if (!m)
1296         return VNET_API_ERROR_NO_SUCH_ENTRY;
1297
1298       if (!is_lb_static_mapping (m))
1299         return VNET_API_ERROR_INVALID_VALUE;
1300
1301       /* Free external address port */
1302       if (!(sm->static_mapping_only || out2in_only))
1303         {
1304           for (i = 0; i < vec_len (sm->addresses); i++)
1305             {
1306               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1307                 {
1308                   a = sm->addresses + i;
1309                   switch (proto)
1310                     {
1311 #define _(N, j, n, s) \
1312                     case SNAT_PROTOCOL_##N: \
1313                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1314                       if (e_port > 1024) \
1315                         { \
1316                           a->busy_##n##_ports--; \
1317                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1318                         } \
1319                       break;
1320                       foreach_snat_protocol
1321 #undef _
1322                     default:
1323                       nat_log_info ("unknown protocol");
1324                       return VNET_API_ERROR_INVALID_VALUE_2;
1325                     }
1326                   break;
1327                 }
1328             }
1329         }
1330
1331       m_key.addr = m->external_addr;
1332       m_key.port = m->external_port;
1333       m_key.protocol = m->proto;
1334       m_key.fib_index = 0;
1335       kv.key = m_key.as_u64;
1336       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1337         {
1338           nat_log_err ("static_mapping_by_external key del failed");
1339           return VNET_API_ERROR_UNSPECIFIED;
1340         }
1341
1342       /* *INDENT-OFF* */
1343       pool_foreach (local, m->locals,
1344       ({
1345           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1346                             FIB_SOURCE_PLUGIN_LOW);
1347           m_key.addr = local->addr;
1348           if (!out2in_only)
1349             {
1350               m_key.port = local->port;
1351               m_key.fib_index = local->fib_index;
1352               kv.key = m_key.as_u64;
1353               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1354                 {
1355                   nat_log_err ("static_mapping_by_local key del failed");
1356                   return VNET_API_ERROR_UNSPECIFIED;
1357                 }
1358             }
1359
1360           if (sm->num_workers > 1)
1361             {
1362               ip4_header_t ip = {
1363                 .src_address = local->addr,
1364               };
1365               tsm = vec_elt_at_index (sm->per_thread_data,
1366                                       sm->worker_in2out_cb (&ip, m->fib_index));
1367             }
1368           else
1369             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1370
1371           /* Delete sessions */
1372           u_key.addr = local->addr;
1373           u_key.fib_index = local->fib_index;
1374           kv.key = u_key.as_u64;
1375           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1376             {
1377               u = pool_elt_at_index (tsm->users, value.value);
1378               if (u->nstaticsessions)
1379                 {
1380                   head_index = u->sessions_per_user_list_head_index;
1381                   head = pool_elt_at_index (tsm->list_pool, head_index);
1382                   elt_index = head->next;
1383                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1384                   ses_index = elt->value;
1385                   while (ses_index != ~0)
1386                     {
1387                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1388                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1389                       ses_index = elt->value;
1390
1391                       if (!(is_lb_session (s)))
1392                         continue;
1393
1394                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1395                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1396                         continue;
1397
1398                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1399                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1400                     }
1401                 }
1402             }
1403       }));
1404       /* *INDENT-ON* */
1405       if (m->affinity)
1406         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1407       pool_free (m->locals);
1408       vec_free (m->tag);
1409       vec_free (m->workers);
1410
1411       pool_put (sm->static_mappings, m);
1412     }
1413
1414   return 0;
1415 }
1416
1417 int
1418 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1419                                        ip4_address_t l_addr, u16 l_port,
1420                                        snat_protocol_t proto, u32 vrf_id,
1421                                        u8 probability, u8 is_add)
1422 {
1423   snat_main_t *sm = &snat_main;
1424   snat_static_mapping_t *m = 0;
1425   snat_session_key_t m_key;
1426   clib_bihash_kv_8_8_t kv, value;
1427   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1428   snat_main_per_thread_data_t *tsm;
1429   snat_user_key_t u_key;
1430   snat_user_t *u;
1431   snat_session_t *s;
1432   dlist_elt_t *head, *elt;
1433   u32 elt_index, head_index, ses_index, *locals = 0;
1434   uword *bitmap = 0;
1435   int i;
1436
1437   if (!sm->endpoint_dependent)
1438     return VNET_API_ERROR_FEATURE_DISABLED;
1439
1440   m_key.addr = e_addr;
1441   m_key.port = e_port;
1442   m_key.protocol = proto;
1443   m_key.fib_index = 0;
1444   kv.key = m_key.as_u64;
1445   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1446     m = pool_elt_at_index (sm->static_mappings, value.value);
1447
1448   if (!m)
1449     return VNET_API_ERROR_NO_SUCH_ENTRY;
1450
1451   if (!is_lb_static_mapping (m))
1452     return VNET_API_ERROR_INVALID_VALUE;
1453
1454   /* *INDENT-OFF* */
1455   pool_foreach (local, m->locals,
1456   ({
1457     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1458         (local->vrf_id == vrf_id))
1459       {
1460         match_local = local;
1461         break;
1462       }
1463   }));
1464   /* *INDENT-ON* */
1465
1466   if (is_add)
1467     {
1468       if (match_local)
1469         return VNET_API_ERROR_VALUE_EXIST;
1470
1471       pool_get (m->locals, local);
1472       clib_memset (local, 0, sizeof (*local));
1473       local->addr.as_u32 = l_addr.as_u32;
1474       local->port = l_port;
1475       local->probability = probability;
1476       local->vrf_id = vrf_id;
1477       local->fib_index =
1478         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1479                                            FIB_SOURCE_PLUGIN_LOW);
1480
1481       if (!is_out2in_only_static_mapping (m))
1482         {
1483           m_key.addr = l_addr;
1484           m_key.port = l_port;
1485           m_key.fib_index = local->fib_index;
1486           kv.key = m_key.as_u64;
1487           kv.value = m - sm->static_mappings;
1488           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1489             nat_log_err ("static_mapping_by_local key add failed");
1490         }
1491     }
1492   else
1493     {
1494       if (!match_local)
1495         return VNET_API_ERROR_NO_SUCH_ENTRY;
1496
1497       if (pool_elts (m->locals) < 3)
1498         return VNET_API_ERROR_UNSPECIFIED;
1499
1500       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1501                         FIB_SOURCE_PLUGIN_LOW);
1502
1503       if (!is_out2in_only_static_mapping (m))
1504         {
1505           m_key.addr = l_addr;
1506           m_key.port = l_port;
1507           m_key.fib_index = match_local->fib_index;
1508           kv.key = m_key.as_u64;
1509           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1510             nat_log_err ("static_mapping_by_local key del failed");
1511         }
1512
1513       if (sm->num_workers > 1)
1514         {
1515           ip4_header_t ip = {
1516             .src_address = local->addr,
1517           };
1518           tsm = vec_elt_at_index (sm->per_thread_data,
1519                                   sm->worker_in2out_cb (&ip, m->fib_index));
1520         }
1521       else
1522         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1523
1524       /* Delete sessions */
1525       u_key.addr = match_local->addr;
1526       u_key.fib_index = match_local->fib_index;
1527       kv.key = u_key.as_u64;
1528       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1529         {
1530           u = pool_elt_at_index (tsm->users, value.value);
1531           if (u->nstaticsessions)
1532             {
1533               head_index = u->sessions_per_user_list_head_index;
1534               head = pool_elt_at_index (tsm->list_pool, head_index);
1535               elt_index = head->next;
1536               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1537               ses_index = elt->value;
1538               while (ses_index != ~0)
1539                 {
1540                   s = pool_elt_at_index (tsm->sessions, ses_index);
1541                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1542                   ses_index = elt->value;
1543
1544                   if (!(is_lb_session (s)))
1545                     continue;
1546
1547                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1548                       (clib_net_to_host_u16 (s->in2out.port) !=
1549                        match_local->port))
1550                     continue;
1551
1552                   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1553                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1554                 }
1555             }
1556         }
1557
1558       pool_put (m->locals, match_local);
1559     }
1560
1561   vec_free (m->workers);
1562
1563   /* *INDENT-OFF* */
1564   pool_foreach (local, m->locals,
1565   ({
1566     vec_add1 (locals, local - m->locals);
1567     if (sm->num_workers > 1)
1568       {
1569         ip4_header_t ip;
1570         ip.src_address.as_u32 = local->addr.as_u32,
1571         bitmap = clib_bitmap_set (bitmap,
1572                                   sm->worker_in2out_cb (&ip, local->fib_index),
1573                                   1);
1574       }
1575   }));
1576   /* *INDENT-ON* */
1577
1578   ASSERT (vec_len (locals) > 1);
1579
1580   local = pool_elt_at_index (m->locals, locals[0]);
1581   local->prefix = local->probability;
1582   for (i = 1; i < vec_len (locals); i++)
1583     {
1584       local = pool_elt_at_index (m->locals, locals[i]);
1585       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1586       local->prefix = local->probability + prev_local->prefix;
1587     }
1588
1589   /* Assign workers */
1590   if (sm->num_workers > 1)
1591     {
1592       /* *INDENT-OFF* */
1593       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1594       /* *INDENT-ON* */
1595     }
1596
1597   return 0;
1598 }
1599
1600 int
1601 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1602                   u8 twice_nat)
1603 {
1604   snat_address_t *a = 0;
1605   snat_session_t *ses;
1606   u32 *ses_to_be_removed = 0, *ses_index;
1607   snat_main_per_thread_data_t *tsm;
1608   snat_static_mapping_t *m;
1609   snat_interface_t *interface;
1610   int i;
1611   snat_address_t *addresses =
1612     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1613
1614   /* Find SNAT address */
1615   for (i = 0; i < vec_len (addresses); i++)
1616     {
1617       if (addresses[i].addr.as_u32 == addr.as_u32)
1618         {
1619           a = addresses + i;
1620           break;
1621         }
1622     }
1623   if (!a)
1624     return VNET_API_ERROR_NO_SUCH_ENTRY;
1625
1626   if (delete_sm)
1627     {
1628       /* *INDENT-OFF* */
1629       pool_foreach (m, sm->static_mappings,
1630       ({
1631           if (m->external_addr.as_u32 == addr.as_u32)
1632             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1633                                             m->local_port, m->external_port,
1634                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1635                                             m->proto, 0, m->twice_nat,
1636                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1637       }));
1638       /* *INDENT-ON* */
1639     }
1640   else
1641     {
1642       /* Check if address is used in some static mapping */
1643       if (is_snat_address_used_in_static_mapping (sm, addr))
1644         {
1645           nat_log_notice ("address used in static mapping");
1646           return VNET_API_ERROR_UNSPECIFIED;
1647         }
1648     }
1649
1650   if (a->fib_index != ~0)
1651     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1652
1653   /* Delete sessions using address */
1654   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1655     {
1656       /* *INDENT-OFF* */
1657       vec_foreach (tsm, sm->per_thread_data)
1658         {
1659           pool_foreach (ses, tsm->sessions, ({
1660             if (ses->out2in.addr.as_u32 == addr.as_u32)
1661               {
1662                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1663                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1664               }
1665           }));
1666
1667           vec_foreach (ses_index, ses_to_be_removed)
1668             {
1669               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1670               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1671             }
1672
1673           vec_free (ses_to_be_removed);
1674         }
1675       /* *INDENT-ON* */
1676     }
1677
1678 #define _(N, i, n, s) \
1679   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1680   vec_free (a->busy_##n##_ports_per_thread);
1681   foreach_snat_protocol
1682 #undef _
1683     if (twice_nat)
1684     {
1685       vec_del1 (sm->twice_nat_addresses, i);
1686       return 0;
1687     }
1688   else
1689     vec_del1 (sm->addresses, i);
1690
1691   /* Delete external address from FIB */
1692   /* *INDENT-OFF* */
1693   pool_foreach (interface, sm->interfaces,
1694   ({
1695     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1696       continue;
1697
1698     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1699     break;
1700   }));
1701   pool_foreach (interface, sm->output_feature_interfaces,
1702   ({
1703     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1704       continue;
1705
1706     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1707     break;
1708   }));
1709   /* *INDENT-ON* */
1710
1711   return 0;
1712 }
1713
1714 int
1715 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1716 {
1717   snat_main_t *sm = &snat_main;
1718   snat_interface_t *i;
1719   const char *feature_name, *del_feature_name;
1720   snat_address_t *ap;
1721   snat_static_mapping_t *m;
1722   snat_det_map_t *dm;
1723   nat_outside_fib_t *outside_fib;
1724   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1725                                                        sw_if_index);
1726
1727   if (sm->out2in_dpo && !is_inside)
1728     return VNET_API_ERROR_UNSUPPORTED;
1729
1730   /* *INDENT-OFF* */
1731   pool_foreach (i, sm->output_feature_interfaces,
1732   ({
1733     if (i->sw_if_index == sw_if_index)
1734       return VNET_API_ERROR_VALUE_EXIST;
1735   }));
1736   /* *INDENT-ON* */
1737
1738   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1739     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1740   else
1741     {
1742       if (sm->num_workers > 1 && !sm->deterministic)
1743         feature_name =
1744           is_inside ? "nat44-in2out-worker-handoff" :
1745           "nat44-out2in-worker-handoff";
1746       else if (sm->deterministic)
1747         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1748       else if (sm->endpoint_dependent)
1749         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1750       else
1751         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1752     }
1753
1754   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1755     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1756                                                       NAT_FQ_NELTS);
1757
1758   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1759     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1760                                                       NAT_FQ_NELTS);
1761
1762   if (!is_inside)
1763     {
1764       /* *INDENT-OFF* */
1765       vec_foreach (outside_fib, sm->outside_fibs)
1766         {
1767           if (outside_fib->fib_index == fib_index)
1768             {
1769               if (is_del)
1770                 {
1771                   outside_fib->refcount--;
1772                   if (!outside_fib->refcount)
1773                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1774                 }
1775               else
1776                 outside_fib->refcount++;
1777               goto feature_set;
1778             }
1779         }
1780       /* *INDENT-ON* */
1781       if (!is_del)
1782         {
1783           vec_add2 (sm->outside_fibs, outside_fib, 1);
1784           outside_fib->refcount = 1;
1785           outside_fib->fib_index = fib_index;
1786         }
1787     }
1788 feature_set:
1789   /* *INDENT-OFF* */
1790   pool_foreach (i, sm->interfaces,
1791   ({
1792     if (i->sw_if_index == sw_if_index)
1793       {
1794         if (is_del)
1795           {
1796             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1797               {
1798                 if (is_inside)
1799                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1800                 else
1801                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1802
1803                 if (sm->num_workers > 1 && !sm->deterministic)
1804                   {
1805                     del_feature_name = "nat44-handoff-classify";
1806                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1807                                                  "nat44-out2in-worker-handoff";
1808                   }
1809                 else if (sm->deterministic)
1810                   {
1811                     del_feature_name = "nat44-det-classify";
1812                     feature_name = !is_inside ?  "nat44-det-in2out" :
1813                                                  "nat44-det-out2in";
1814                   }
1815                 else if (sm->endpoint_dependent)
1816                   {
1817                     del_feature_name = "nat44-ed-classify";
1818                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1819                                                  "nat44-ed-out2in";
1820                   }
1821                 else
1822                   {
1823                     del_feature_name = "nat44-classify";
1824                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1825                   }
1826
1827                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1828                                              sw_if_index, 0, 0, 0);
1829                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1830                                              sw_if_index, 1, 0, 0);
1831                 if (!is_inside)
1832                   {
1833                     if (sm->endpoint_dependent)
1834                       vnet_feature_enable_disable ("ip4-local",
1835                                                    "nat44-ed-hairpinning",
1836                                                    sw_if_index, 1, 0, 0);
1837                     else if (!sm->deterministic)
1838                       vnet_feature_enable_disable ("ip4-local",
1839                                                    "nat44-hairpinning",
1840                                                    sw_if_index, 1, 0, 0);
1841                   }
1842               }
1843             else
1844               {
1845                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1846                                              sw_if_index, 0, 0, 0);
1847                 pool_put (sm->interfaces, i);
1848                 if (is_inside)
1849                   {
1850                     if (sm->endpoint_dependent)
1851                       vnet_feature_enable_disable ("ip4-local",
1852                                                    "nat44-ed-hairpinning",
1853                                                    sw_if_index, 0, 0, 0);
1854                     else if (!sm->deterministic)
1855                       vnet_feature_enable_disable ("ip4-local",
1856                                                    "nat44-hairpinning",
1857                                                    sw_if_index, 0, 0, 0);
1858                   }
1859               }
1860           }
1861         else
1862           {
1863             if ((nat_interface_is_inside(i) && is_inside) ||
1864                 (nat_interface_is_outside(i) && !is_inside))
1865               return 0;
1866
1867             if (sm->num_workers > 1 && !sm->deterministic)
1868               {
1869                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1870                                                  "nat44-out2in-worker-handoff";
1871                 feature_name = "nat44-handoff-classify";
1872               }
1873             else if (sm->deterministic)
1874               {
1875                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1876                                                  "nat44-det-out2in";
1877                 feature_name = "nat44-det-classify";
1878               }
1879             else if (sm->endpoint_dependent)
1880               {
1881                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1882                                                  "nat44-ed-out2in";
1883                 feature_name = "nat44-ed-classify";
1884               }
1885             else
1886               {
1887                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1888                 feature_name = "nat44-classify";
1889               }
1890
1891             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1892                                          sw_if_index, 0, 0, 0);
1893             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1894                                          sw_if_index, 1, 0, 0);
1895             if (!is_inside)
1896               {
1897                 if (sm->endpoint_dependent)
1898                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1899                                                sw_if_index, 0, 0, 0);
1900                 else if (!sm->deterministic)
1901                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1902                                                sw_if_index, 0, 0, 0);
1903               }
1904             goto set_flags;
1905           }
1906
1907         goto fib;
1908       }
1909   }));
1910   /* *INDENT-ON* */
1911
1912   if (is_del)
1913     return VNET_API_ERROR_NO_SUCH_ENTRY;
1914
1915   pool_get (sm->interfaces, i);
1916   i->sw_if_index = sw_if_index;
1917   i->flags = 0;
1918   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1919                                0);
1920
1921   if (is_inside && !sm->out2in_dpo)
1922     {
1923       if (sm->endpoint_dependent)
1924         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1925                                      sw_if_index, 1, 0, 0);
1926       else if (!sm->deterministic)
1927         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1928                                      sw_if_index, 1, 0, 0);
1929     }
1930
1931 set_flags:
1932   if (is_inside)
1933     {
1934       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1935       return 0;
1936     }
1937   else
1938     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1939
1940   /* Add/delete external addresses to FIB */
1941 fib:
1942   /* *INDENT-OFF* */
1943   vec_foreach (ap, sm->addresses)
1944     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1945
1946   pool_foreach (m, sm->static_mappings,
1947   ({
1948     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1949       continue;
1950
1951     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1952   }));
1953
1954   pool_foreach (dm, sm->det_maps,
1955   ({
1956     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1957   }));
1958   /* *INDENT-ON* */
1959
1960   return 0;
1961 }
1962
1963 int
1964 snat_interface_add_del_output_feature (u32 sw_if_index,
1965                                        u8 is_inside, int is_del)
1966 {
1967   snat_main_t *sm = &snat_main;
1968   snat_interface_t *i;
1969   snat_address_t *ap;
1970   snat_static_mapping_t *m;
1971   nat_outside_fib_t *outside_fib;
1972   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1973                                                        sw_if_index);
1974
1975
1976   if (sm->deterministic ||
1977       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1978     return VNET_API_ERROR_UNSUPPORTED;
1979
1980   /* *INDENT-OFF* */
1981   pool_foreach (i, sm->interfaces,
1982   ({
1983     if (i->sw_if_index == sw_if_index)
1984       return VNET_API_ERROR_VALUE_EXIST;
1985   }));
1986   /* *INDENT-ON* */
1987
1988   if (!is_inside)
1989     {
1990       /* *INDENT-OFF* */
1991       vec_foreach (outside_fib, sm->outside_fibs)
1992         {
1993           if (outside_fib->fib_index == fib_index)
1994             {
1995               if (is_del)
1996                 {
1997                   outside_fib->refcount--;
1998                   if (!outside_fib->refcount)
1999                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2000                 }
2001               else
2002                 outside_fib->refcount++;
2003               goto feature_set;
2004             }
2005         }
2006       /* *INDENT-ON* */
2007       if (!is_del)
2008         {
2009           vec_add2 (sm->outside_fibs, outside_fib, 1);
2010           outside_fib->refcount = 1;
2011           outside_fib->fib_index = fib_index;
2012         }
2013     }
2014
2015 feature_set:
2016   if (is_inside)
2017     {
2018       if (sm->endpoint_dependent)
2019         {
2020           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2021                                        sw_if_index, !is_del, 0, 0);
2022           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2023                                        sw_if_index, !is_del, 0, 0);
2024         }
2025       else
2026         {
2027           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2028                                        sw_if_index, !is_del, 0, 0);
2029           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2030                                        sw_if_index, !is_del, 0, 0);
2031         }
2032       goto fq;
2033     }
2034
2035   if (sm->num_workers > 1)
2036     {
2037       vnet_feature_enable_disable ("ip4-unicast",
2038                                    "nat44-out2in-worker-handoff",
2039                                    sw_if_index, !is_del, 0, 0);
2040       vnet_feature_enable_disable ("ip4-output",
2041                                    "nat44-in2out-output-worker-handoff",
2042                                    sw_if_index, !is_del, 0, 0);
2043     }
2044   else
2045     {
2046       if (sm->endpoint_dependent)
2047         {
2048           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2049                                        sw_if_index, !is_del, 0, 0);
2050           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2051                                        sw_if_index, !is_del, 0, 0);
2052         }
2053       else
2054         {
2055           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2056                                        sw_if_index, !is_del, 0, 0);
2057           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2058                                        sw_if_index, !is_del, 0, 0);
2059         }
2060     }
2061
2062 fq:
2063   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2064     sm->fq_in2out_output_index =
2065       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2066
2067   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2068     sm->fq_out2in_index =
2069       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2070
2071   /* *INDENT-OFF* */
2072   pool_foreach (i, sm->output_feature_interfaces,
2073   ({
2074     if (i->sw_if_index == sw_if_index)
2075       {
2076         if (is_del)
2077           pool_put (sm->output_feature_interfaces, i);
2078         else
2079           return VNET_API_ERROR_VALUE_EXIST;
2080
2081         goto fib;
2082       }
2083   }));
2084   /* *INDENT-ON* */
2085
2086   if (is_del)
2087     return VNET_API_ERROR_NO_SUCH_ENTRY;
2088
2089   pool_get (sm->output_feature_interfaces, i);
2090   i->sw_if_index = sw_if_index;
2091   i->flags = 0;
2092   if (is_inside)
2093     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2094   else
2095     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2096
2097   /* Add/delete external addresses to FIB */
2098 fib:
2099   if (is_inside)
2100     return 0;
2101
2102   /* *INDENT-OFF* */
2103   vec_foreach (ap, sm->addresses)
2104     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2105
2106   pool_foreach (m, sm->static_mappings,
2107   ({
2108     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2109       continue;
2110
2111     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2112   }));
2113   /* *INDENT-ON* */
2114
2115   return 0;
2116 }
2117
2118 int
2119 snat_set_workers (uword * bitmap)
2120 {
2121   snat_main_t *sm = &snat_main;
2122   int i, j = 0;
2123
2124   if (sm->num_workers < 2)
2125     return VNET_API_ERROR_FEATURE_DISABLED;
2126
2127   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2128     return VNET_API_ERROR_INVALID_WORKER;
2129
2130   vec_free (sm->workers);
2131   /* *INDENT-OFF* */
2132   clib_bitmap_foreach (i, bitmap,
2133     ({
2134       vec_add1(sm->workers, i);
2135       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2136       j++;
2137     }));
2138   /* *INDENT-ON* */
2139
2140   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2141   sm->num_snat_thread = _vec_len (sm->workers);
2142
2143   return 0;
2144 }
2145
2146
2147 static void
2148 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2149                                        uword opaque,
2150                                        u32 sw_if_index,
2151                                        ip4_address_t * address,
2152                                        u32 address_length,
2153                                        u32 if_address_index, u32 is_delete);
2154
2155 static void
2156 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2157                                  uword opaque,
2158                                  u32 sw_if_index,
2159                                  ip4_address_t * address,
2160                                  u32 address_length,
2161                                  u32 if_address_index, u32 is_delete);
2162
2163 static int
2164 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2165                                  u32 fib_index,
2166                                  u32 thread_index,
2167                                  snat_session_key_t * k,
2168                                  u16 port_per_thread, u32 snat_thread_index);
2169
2170 static clib_error_t *
2171 snat_init (vlib_main_t * vm)
2172 {
2173   snat_main_t *sm = &snat_main;
2174   clib_error_t *error = 0;
2175   ip4_main_t *im = &ip4_main;
2176   ip_lookup_main_t *lm = &im->lookup_main;
2177   uword *p;
2178   vlib_thread_registration_t *tr;
2179   vlib_thread_main_t *tm = vlib_get_thread_main ();
2180   uword *bitmap = 0;
2181   u32 i;
2182   ip4_add_del_interface_address_callback_t cb4;
2183   vlib_node_t *error_drop_node;
2184
2185   sm->vlib_main = vm;
2186   sm->vnet_main = vnet_get_main ();
2187   sm->ip4_main = im;
2188   sm->ip4_lookup_main = lm;
2189   sm->api_main = &api_main;
2190   sm->first_worker_index = 0;
2191   sm->num_workers = 0;
2192   sm->num_snat_thread = 1;
2193   sm->workers = 0;
2194   sm->port_per_thread = 0xffff - 1024;
2195   sm->fq_in2out_index = ~0;
2196   sm->fq_out2in_index = ~0;
2197   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2198   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2199   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2200   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2201   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2202   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2203   sm->forwarding_enabled = 0;
2204   sm->log_class = vlib_log_register_class ("nat", 0);
2205   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2206   sm->error_node_index = error_drop_node->index;
2207   sm->mss_clamping = 0;
2208
2209   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2210   if (p)
2211     {
2212       tr = (vlib_thread_registration_t *) p[0];
2213       if (tr)
2214         {
2215           sm->num_workers = tr->count;
2216           sm->first_worker_index = tr->first_index;
2217         }
2218     }
2219
2220   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2221
2222   /* Use all available workers by default */
2223   if (sm->num_workers > 1)
2224     {
2225       for (i = 0; i < sm->num_workers; i++)
2226         bitmap = clib_bitmap_set (bitmap, i, 1);
2227       snat_set_workers (bitmap);
2228       clib_bitmap_free (bitmap);
2229     }
2230   else
2231     {
2232       sm->per_thread_data[0].snat_thread_index = 0;
2233     }
2234
2235   error = snat_api_init (vm, sm);
2236   if (error)
2237     return error;
2238
2239   /* Set up the interface address add/del callback */
2240   cb4.function = snat_ip4_add_del_interface_address_cb;
2241   cb4.function_opaque = 0;
2242
2243   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2244
2245   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2246   cb4.function_opaque = 0;
2247
2248   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2249
2250   nat_dpo_module_init ();
2251
2252   /* Init counters */
2253   sm->total_users.name = "total-users";
2254   sm->total_users.stat_segment_name = "/nat44/total-users";
2255   vlib_validate_simple_counter (&sm->total_users, 0);
2256   vlib_zero_simple_counter (&sm->total_users, 0);
2257   sm->total_sessions.name = "total-sessions";
2258   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2259   vlib_validate_simple_counter (&sm->total_sessions, 0);
2260   vlib_zero_simple_counter (&sm->total_sessions, 0);
2261
2262   /* Init IPFIX logging */
2263   snat_ipfix_logging_init (vm);
2264
2265   /* Init NAT64 */
2266   error = nat64_init (vm);
2267   if (error)
2268     return error;
2269
2270   dslite_init (vm);
2271
2272   nat66_init ();
2273
2274   /* Init virtual fragmenentation reassembly */
2275   return nat_reass_init (vm);
2276 }
2277
2278 VLIB_INIT_FUNCTION (snat_init);
2279
2280 void
2281 snat_free_outside_address_and_port (snat_address_t * addresses,
2282                                     u32 thread_index, snat_session_key_t * k)
2283 {
2284   snat_address_t *a;
2285   u32 address_index;
2286   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2287
2288   for (address_index = 0; address_index < vec_len (addresses);
2289        address_index++)
2290     {
2291       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2292         break;
2293     }
2294
2295   ASSERT (address_index < vec_len (addresses));
2296
2297   a = addresses + address_index;
2298
2299   switch (k->protocol)
2300     {
2301 #define _(N, i, n, s) \
2302     case SNAT_PROTOCOL_##N: \
2303       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2304         port_host_byte_order) == 1); \
2305       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2306         port_host_byte_order, 0); \
2307       a->busy_##n##_ports--; \
2308       a->busy_##n##_ports_per_thread[thread_index]--; \
2309       break;
2310       foreach_snat_protocol
2311 #undef _
2312     default:
2313       nat_log_info ("unknown protocol");
2314       return;
2315     }
2316 }
2317
2318 int
2319 snat_static_mapping_match (snat_main_t * sm,
2320                            snat_session_key_t match,
2321                            snat_session_key_t * mapping,
2322                            u8 by_external,
2323                            u8 * is_addr_only,
2324                            twice_nat_type_t * twice_nat,
2325                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2326                            u8 * is_identity_nat)
2327 {
2328   clib_bihash_kv_8_8_t kv, value;
2329   snat_static_mapping_t *m;
2330   snat_session_key_t m_key;
2331   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2332   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2333   u8 backend_index;
2334   nat44_lb_addr_port_t *local;
2335
2336   m_key.fib_index = match.fib_index;
2337   if (by_external)
2338     {
2339       mapping_hash = &sm->static_mapping_by_external;
2340       m_key.fib_index = 0;
2341     }
2342
2343   m_key.addr = match.addr;
2344   m_key.port = clib_net_to_host_u16 (match.port);
2345   m_key.protocol = match.protocol;
2346
2347   kv.key = m_key.as_u64;
2348
2349   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2350     {
2351       /* Try address only mapping */
2352       m_key.port = 0;
2353       m_key.protocol = 0;
2354       kv.key = m_key.as_u64;
2355       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2356         return 1;
2357     }
2358
2359   m = pool_elt_at_index (sm->static_mappings, value.value);
2360
2361   if (by_external)
2362     {
2363       if (is_lb_static_mapping (m))
2364         {
2365           if (PREDICT_FALSE (lb != 0))
2366             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2367           if (m->affinity)
2368             {
2369               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2370                                               match.protocol, match.port,
2371                                               &backend_index))
2372                 goto get_local;
2373
2374               local = pool_elt_at_index (m->locals, backend_index);
2375               mapping->addr = local->addr;
2376               mapping->port = clib_host_to_net_u16 (local->port);
2377               mapping->fib_index = local->fib_index;
2378               goto end;
2379             }
2380         get_local:
2381           /* *INDENT-OFF* */
2382           pool_foreach_index (i, m->locals,
2383           ({
2384             vec_add1 (tmp, i);
2385           }));
2386           /* *INDENT-ON* */
2387           hi = vec_len (tmp) - 1;
2388           local = pool_elt_at_index (m->locals, tmp[hi]);
2389           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2390           while (lo < hi)
2391             {
2392               mid = ((hi - lo) >> 1) + lo;
2393               local = pool_elt_at_index (m->locals, tmp[mid]);
2394               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2395             }
2396           local = pool_elt_at_index (m->locals, tmp[lo]);
2397           if (!(local->prefix >= rand))
2398             return 1;
2399           if (PREDICT_FALSE (sm->num_workers > 1))
2400             {
2401               ip4_header_t ip = {
2402                 .src_address = local->addr,
2403               };
2404               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2405                   vlib_get_thread_index ())
2406                 goto get_local;
2407             }
2408           mapping->addr = local->addr;
2409           mapping->port = clib_host_to_net_u16 (local->port);
2410           mapping->fib_index = local->fib_index;
2411           if (m->affinity)
2412             {
2413               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2414                                                 match.protocol, match.port,
2415                                                 tmp[lo], m->affinity,
2416                                                 m->affinity_per_service_list_head_index))
2417                 nat_log_info ("create affinity record failed");
2418             }
2419           vec_free (tmp);
2420         }
2421       else
2422         {
2423           if (PREDICT_FALSE (lb != 0))
2424             *lb = NO_LB_NAT;
2425           mapping->fib_index = m->fib_index;
2426           mapping->addr = m->local_addr;
2427           /* Address only mapping doesn't change port */
2428           mapping->port = is_addr_only_static_mapping (m) ? match.port
2429             : clib_host_to_net_u16 (m->local_port);
2430         }
2431       mapping->protocol = m->proto;
2432     }
2433   else
2434     {
2435       mapping->addr = m->external_addr;
2436       /* Address only mapping doesn't change port */
2437       mapping->port = is_addr_only_static_mapping (m) ? match.port
2438         : clib_host_to_net_u16 (m->external_port);
2439       mapping->fib_index = sm->outside_fib_index;
2440     }
2441
2442 end:
2443   if (PREDICT_FALSE (is_addr_only != 0))
2444     *is_addr_only = is_addr_only_static_mapping (m);
2445
2446   if (PREDICT_FALSE (twice_nat != 0))
2447     *twice_nat = m->twice_nat;
2448
2449   if (PREDICT_FALSE (is_identity_nat != 0))
2450     *is_identity_nat = is_identity_static_mapping (m);
2451
2452   return 0;
2453 }
2454
2455 static_always_inline u16
2456 snat_random_port (u16 min, u16 max)
2457 {
2458   snat_main_t *sm = &snat_main;
2459   return min + random_u32 (&sm->random_seed) /
2460     (random_u32_max () / (max - min + 1) + 1);
2461 }
2462
2463 int
2464 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2465                                      u32 fib_index,
2466                                      u32 thread_index,
2467                                      snat_session_key_t * k,
2468                                      u16 port_per_thread,
2469                                      u32 snat_thread_index)
2470 {
2471   snat_main_t *sm = &snat_main;
2472
2473   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2474                                   port_per_thread, snat_thread_index);
2475 }
2476
2477 static int
2478 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2479                                  u32 fib_index,
2480                                  u32 thread_index,
2481                                  snat_session_key_t * k,
2482                                  u16 port_per_thread, u32 snat_thread_index)
2483 {
2484   int i;
2485   snat_address_t *a, *ga = 0;
2486   u32 portnum;
2487
2488   for (i = 0; i < vec_len (addresses); i++)
2489     {
2490       a = addresses + i;
2491       switch (k->protocol)
2492         {
2493 #define _(N, j, n, s) \
2494         case SNAT_PROTOCOL_##N: \
2495           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2496             { \
2497               if (a->fib_index == fib_index) \
2498                 { \
2499                   while (1) \
2500                     { \
2501                       portnum = (port_per_thread * \
2502                         snat_thread_index) + \
2503                         snat_random_port(1, port_per_thread) + 1024; \
2504                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2505                         continue; \
2506                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2507                       a->busy_##n##_ports_per_thread[thread_index]++; \
2508                       a->busy_##n##_ports++; \
2509                       k->addr = a->addr; \
2510                       k->port = clib_host_to_net_u16(portnum); \
2511                       return 0; \
2512                     } \
2513                 } \
2514               else if (a->fib_index == ~0) \
2515                 { \
2516                   ga = a; \
2517                 } \
2518             } \
2519           break;
2520           foreach_snat_protocol
2521 #undef _
2522         default:
2523           nat_log_info ("unknown protocol");
2524           return 1;
2525         }
2526
2527     }
2528
2529   if (ga)
2530     {
2531       a = ga;
2532       switch (k->protocol)
2533         {
2534 #define _(N, j, n, s) \
2535         case SNAT_PROTOCOL_##N: \
2536           while (1) \
2537             { \
2538               portnum = (port_per_thread * \
2539                 snat_thread_index) + \
2540                 snat_random_port(1, port_per_thread) + 1024; \
2541               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2542                 continue; \
2543               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2544               a->busy_##n##_ports_per_thread[thread_index]++; \
2545               a->busy_##n##_ports++; \
2546               k->addr = a->addr; \
2547               k->port = clib_host_to_net_u16(portnum); \
2548               return 0; \
2549             }
2550           break;
2551           foreach_snat_protocol
2552 #undef _
2553         default:
2554           nat_log_info ("unknown protocol");
2555           return 1;
2556         }
2557     }
2558
2559   /* Totally out of translations to use... */
2560   snat_ipfix_logging_addresses_exhausted (0);
2561   return 1;
2562 }
2563
2564 static int
2565 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2566                               u32 fib_index,
2567                               u32 thread_index,
2568                               snat_session_key_t * k,
2569                               u16 port_per_thread, u32 snat_thread_index)
2570 {
2571   snat_main_t *sm = &snat_main;
2572   snat_address_t *a = addresses;
2573   u16 m, ports, portnum, A, j;
2574   m = 16 - (sm->psid_offset + sm->psid_length);
2575   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2576
2577   if (!vec_len (addresses))
2578     goto exhausted;
2579
2580   switch (k->protocol)
2581     {
2582 #define _(N, i, n, s) \
2583     case SNAT_PROTOCOL_##N: \
2584       if (a->busy_##n##_ports < ports) \
2585         { \
2586           while (1) \
2587             { \
2588               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2589               j = snat_random_port(0, pow2_mask(m)); \
2590               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2591               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2592                 continue; \
2593               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2594               a->busy_##n##_ports++; \
2595               k->addr = a->addr; \
2596               k->port = clib_host_to_net_u16 (portnum); \
2597               return 0; \
2598             } \
2599         } \
2600       break;
2601       foreach_snat_protocol
2602 #undef _
2603     default:
2604       nat_log_info ("unknown protocol");
2605       return 1;
2606     }
2607
2608 exhausted:
2609   /* Totally out of translations to use... */
2610   snat_ipfix_logging_addresses_exhausted (0);
2611   return 1;
2612 }
2613
2614 static int
2615 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2616                                u32 fib_index,
2617                                u32 thread_index,
2618                                snat_session_key_t * k,
2619                                u16 port_per_thread, u32 snat_thread_index)
2620 {
2621   snat_main_t *sm = &snat_main;
2622   snat_address_t *a = addresses;
2623   u16 portnum, ports;
2624
2625   ports = sm->end_port - sm->start_port + 1;
2626
2627   if (!vec_len (addresses))
2628     goto exhausted;
2629
2630   switch (k->protocol)
2631     {
2632 #define _(N, i, n, s) \
2633     case SNAT_PROTOCOL_##N: \
2634       if (a->busy_##n##_ports < ports) \
2635         { \
2636           while (1) \
2637             { \
2638               portnum = snat_random_port(sm->start_port, sm->end_port); \
2639               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2640                 continue; \
2641               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2642               a->busy_##n##_ports++; \
2643               k->addr = a->addr; \
2644               k->port = clib_host_to_net_u16 (portnum); \
2645               return 0; \
2646             } \
2647         } \
2648       break;
2649       foreach_snat_protocol
2650 #undef _
2651     default:
2652       nat_log_info ("unknown protocol");
2653       return 1;
2654     }
2655
2656 exhausted:
2657   /* Totally out of translations to use... */
2658   snat_ipfix_logging_addresses_exhausted (0);
2659   return 1;
2660 }
2661
2662 void
2663 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2664 {
2665   dpo_id_t dpo_v4 = DPO_INVALID;
2666   fib_prefix_t pfx = {
2667     .fp_proto = FIB_PROTOCOL_IP4,
2668     .fp_len = 32,
2669     .fp_addr.ip4.as_u32 = addr.as_u32,
2670   };
2671
2672   if (is_add)
2673     {
2674       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2675       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2676                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2677       dpo_reset (&dpo_v4);
2678     }
2679   else
2680     {
2681       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2682     }
2683 }
2684
2685 u8 *
2686 format_session_kvp (u8 * s, va_list * args)
2687 {
2688   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2689   snat_session_key_t k;
2690
2691   k.as_u64 = v->key;
2692
2693   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2694
2695   return s;
2696 }
2697
2698 u8 *
2699 format_static_mapping_kvp (u8 * s, va_list * args)
2700 {
2701   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2702   snat_session_key_t k;
2703
2704   k.as_u64 = v->key;
2705
2706   s = format (s, "%U static-mapping-index %llu",
2707               format_static_mapping_key, &k, v->value);
2708
2709   return s;
2710 }
2711
2712 u8 *
2713 format_user_kvp (u8 * s, va_list * args)
2714 {
2715   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2716   snat_user_key_t k;
2717
2718   k.as_u64 = v->key;
2719
2720   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2721               k.fib_index, v->value);
2722
2723   return s;
2724 }
2725
2726 u8 *
2727 format_ed_session_kvp (u8 * s, va_list * args)
2728 {
2729   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2730   nat_ed_ses_key_t k;
2731
2732   k.as_u64[0] = v->key[0];
2733   k.as_u64[1] = v->key[1];
2734
2735   s =
2736     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2737             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2738             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2739             format_ip_protocol, k.proto, k.fib_index, v->value);
2740
2741   return s;
2742 }
2743
2744 static u32
2745 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2746 {
2747   snat_main_t *sm = &snat_main;
2748   u32 next_worker_index = 0;
2749   u32 hash;
2750
2751   next_worker_index = sm->first_worker_index;
2752   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2753     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2754
2755   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2756     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2757   else
2758     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2759
2760   return next_worker_index;
2761 }
2762
2763 static u32
2764 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2765 {
2766   snat_main_t *sm = &snat_main;
2767   udp_header_t *udp;
2768   u16 port;
2769   snat_session_key_t m_key;
2770   clib_bihash_kv_8_8_t kv, value;
2771   snat_static_mapping_t *m;
2772   u32 proto;
2773   u32 next_worker_index = 0;
2774
2775   /* first try static mappings without port */
2776   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2777     {
2778       m_key.addr = ip0->dst_address;
2779       m_key.port = 0;
2780       m_key.protocol = 0;
2781       m_key.fib_index = rx_fib_index0;
2782       kv.key = m_key.as_u64;
2783       if (!clib_bihash_search_8_8
2784           (&sm->static_mapping_by_external, &kv, &value))
2785         {
2786           m = pool_elt_at_index (sm->static_mappings, value.value);
2787           return m->workers[0];
2788         }
2789     }
2790
2791   proto = ip_proto_to_snat_proto (ip0->protocol);
2792   udp = ip4_next_header (ip0);
2793   port = udp->dst_port;
2794
2795   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2796     {
2797       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2798         return vlib_get_thread_index ();
2799
2800       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2801         {
2802           nat_reass_ip4_t *reass;
2803
2804           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2805                                       ip0->fragment_id, ip0->protocol);
2806
2807           if (reass && (reass->thread_index != (u32) ~ 0))
2808             return reass->thread_index;
2809           else
2810             return vlib_get_thread_index ();
2811         }
2812     }
2813
2814   /* unknown protocol */
2815   if (PREDICT_FALSE (proto == ~0))
2816     {
2817       /* use current thread */
2818       return vlib_get_thread_index ();
2819     }
2820
2821   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2822     {
2823       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2824       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2825       if (!icmp_is_error_message (icmp))
2826         port = echo->identifier;
2827       else
2828         {
2829           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2830           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2831           void *l4_header = ip4_next_header (inner_ip);
2832           switch (proto)
2833             {
2834             case SNAT_PROTOCOL_ICMP:
2835               icmp = (icmp46_header_t *) l4_header;
2836               echo = (icmp_echo_header_t *) (icmp + 1);
2837               port = echo->identifier;
2838               break;
2839             case SNAT_PROTOCOL_UDP:
2840             case SNAT_PROTOCOL_TCP:
2841               port = ((tcp_udp_header_t *) l4_header)->src_port;
2842               break;
2843             default:
2844               return vlib_get_thread_index ();
2845             }
2846         }
2847     }
2848
2849   /* try static mappings with port */
2850   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2851     {
2852       m_key.addr = ip0->dst_address;
2853       m_key.port = clib_net_to_host_u16 (port);
2854       m_key.protocol = proto;
2855       m_key.fib_index = rx_fib_index0;
2856       kv.key = m_key.as_u64;
2857       if (!clib_bihash_search_8_8
2858           (&sm->static_mapping_by_external, &kv, &value))
2859         {
2860           m = pool_elt_at_index (sm->static_mappings, value.value);
2861           return m->workers[0];
2862         }
2863     }
2864
2865   /* worker by outside port */
2866   next_worker_index = sm->first_worker_index;
2867   next_worker_index +=
2868     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2869   return next_worker_index;
2870 }
2871
2872 static u32
2873 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2874 {
2875   snat_main_t *sm = &snat_main;
2876   clib_bihash_kv_8_8_t kv, value;
2877   u32 proto, next_worker_index = 0;
2878   udp_header_t *udp;
2879   u16 port;
2880   snat_static_mapping_t *m;
2881   u32 hash;
2882
2883   /* first try static mappings without port */
2884   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2885     {
2886       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2887       if (!clib_bihash_search_8_8
2888           (&sm->static_mapping_by_external, &kv, &value))
2889         {
2890           m = pool_elt_at_index (sm->static_mappings, value.value);
2891           return m->workers[0];
2892         }
2893     }
2894
2895   proto = ip_proto_to_snat_proto (ip->protocol);
2896
2897   /* unknown protocol */
2898   if (PREDICT_FALSE (proto == ~0))
2899     {
2900       /* use current thread */
2901       return vlib_get_thread_index ();
2902     }
2903
2904   udp = ip4_next_header (ip);
2905   port = udp->dst_port;
2906
2907   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2908     {
2909       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2910       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2911       if (!icmp_is_error_message (icmp))
2912         port = echo->identifier;
2913       else
2914         {
2915           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2916           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2917           void *l4_header = ip4_next_header (inner_ip);
2918           switch (proto)
2919             {
2920             case SNAT_PROTOCOL_ICMP:
2921               icmp = (icmp46_header_t *) l4_header;
2922               echo = (icmp_echo_header_t *) (icmp + 1);
2923               port = echo->identifier;
2924               break;
2925             case SNAT_PROTOCOL_UDP:
2926             case SNAT_PROTOCOL_TCP:
2927               port = ((tcp_udp_header_t *) l4_header)->src_port;
2928               break;
2929             default:
2930               return vlib_get_thread_index ();
2931             }
2932         }
2933     }
2934
2935   /* try static mappings with port */
2936   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2937     {
2938       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2939                   clib_net_to_host_u16 (port));
2940       if (!clib_bihash_search_8_8
2941           (&sm->static_mapping_by_external, &kv, &value))
2942         {
2943           m = pool_elt_at_index (sm->static_mappings, value.value);
2944           if (!is_lb_static_mapping (m))
2945             return m->workers[0];
2946
2947           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2948             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
2949
2950           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2951             return m->workers[hash & (_vec_len (m->workers) - 1)];
2952           else
2953             return m->workers[hash % _vec_len (m->workers)];
2954         }
2955     }
2956
2957   /* worker by outside port */
2958   next_worker_index = sm->first_worker_index;
2959   next_worker_index +=
2960     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2961
2962   return next_worker_index;
2963 }
2964
2965 static clib_error_t *
2966 snat_config (vlib_main_t * vm, unformat_input_t * input)
2967 {
2968   snat_main_t *sm = &snat_main;
2969   nat66_main_t *nm = &nat66_main;
2970   u32 translation_buckets = 1024;
2971   u32 translation_memory_size = 128 << 20;
2972   u32 user_buckets = 128;
2973   u32 user_memory_size = 64 << 20;
2974   u32 max_translations_per_user = 100;
2975   u32 outside_vrf_id = 0;
2976   u32 outside_ip6_vrf_id = 0;
2977   u32 inside_vrf_id = 0;
2978   u32 static_mapping_buckets = 1024;
2979   u32 static_mapping_memory_size = 64 << 20;
2980   u32 nat64_bib_buckets = 1024;
2981   u32 nat64_bib_memory_size = 128 << 20;
2982   u32 nat64_st_buckets = 2048;
2983   u32 nat64_st_memory_size = 256 << 20;
2984   u8 static_mapping_only = 0;
2985   u8 static_mapping_connection_tracking = 0;
2986   snat_main_per_thread_data_t *tsm;
2987   dslite_main_t *dm = &dslite_main;
2988
2989   sm->deterministic = 0;
2990   sm->out2in_dpo = 0;
2991   sm->endpoint_dependent = 0;
2992
2993   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2994     {
2995       if (unformat
2996           (input, "translation hash buckets %d", &translation_buckets))
2997         ;
2998       else if (unformat (input, "translation hash memory %d",
2999                          &translation_memory_size));
3000       else if (unformat (input, "user hash buckets %d", &user_buckets))
3001         ;
3002       else if (unformat (input, "user hash memory %d", &user_memory_size))
3003         ;
3004       else if (unformat (input, "max translations per user %d",
3005                          &max_translations_per_user))
3006         ;
3007       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3008         ;
3009       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3010         ;
3011       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3012         ;
3013       else if (unformat (input, "static mapping only"))
3014         {
3015           static_mapping_only = 1;
3016           if (unformat (input, "connection tracking"))
3017             static_mapping_connection_tracking = 1;
3018         }
3019       else if (unformat (input, "deterministic"))
3020         sm->deterministic = 1;
3021       else if (unformat (input, "nat64 bib hash buckets %d",
3022                          &nat64_bib_buckets))
3023         ;
3024       else if (unformat (input, "nat64 bib hash memory %d",
3025                          &nat64_bib_memory_size))
3026         ;
3027       else
3028         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3029         ;
3030       else if (unformat (input, "nat64 st hash memory %d",
3031                          &nat64_st_memory_size))
3032         ;
3033       else if (unformat (input, "out2in dpo"))
3034         sm->out2in_dpo = 1;
3035       else if (unformat (input, "dslite ce"))
3036         dslite_set_ce (dm, 1);
3037       else if (unformat (input, "endpoint-dependent"))
3038         sm->endpoint_dependent = 1;
3039       else
3040         return clib_error_return (0, "unknown input '%U'",
3041                                   format_unformat_error, input);
3042     }
3043
3044   if (sm->deterministic && sm->endpoint_dependent)
3045     return clib_error_return (0,
3046                               "deterministic and endpoint-dependent modes are mutually exclusive");
3047
3048   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3049     return clib_error_return (0,
3050                               "static mapping only mode available only for simple nat");
3051
3052   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3053     return clib_error_return (0,
3054                               "out2in dpo mode available only for simple nat");
3055
3056   /* for show commands, etc. */
3057   sm->translation_buckets = translation_buckets;
3058   sm->translation_memory_size = translation_memory_size;
3059   /* do not exceed load factor 10 */
3060   sm->max_translations = 10 * translation_buckets;
3061   sm->user_buckets = user_buckets;
3062   sm->user_memory_size = user_memory_size;
3063   sm->max_translations_per_user = max_translations_per_user;
3064   sm->outside_vrf_id = outside_vrf_id;
3065   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3066                                                              outside_vrf_id,
3067                                                              FIB_SOURCE_PLUGIN_HI);
3068   nm->outside_vrf_id = outside_ip6_vrf_id;
3069   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3070                                                              outside_ip6_vrf_id,
3071                                                              FIB_SOURCE_PLUGIN_HI);
3072   sm->inside_vrf_id = inside_vrf_id;
3073   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3074                                                             inside_vrf_id,
3075                                                             FIB_SOURCE_PLUGIN_HI);
3076   sm->static_mapping_only = static_mapping_only;
3077   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3078
3079   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3080                   nat64_st_memory_size);
3081
3082   if (sm->deterministic)
3083     {
3084       sm->in2out_node_index = snat_det_in2out_node.index;
3085       sm->in2out_output_node_index = ~0;
3086       sm->out2in_node_index = snat_det_out2in_node.index;
3087       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3088       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3089     }
3090   else
3091     {
3092       if (sm->endpoint_dependent)
3093         {
3094           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3095           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3096           sm->in2out_node_index = nat44_ed_in2out_node.index;
3097           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3098           sm->out2in_node_index = nat44_ed_out2in_node.index;
3099           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3100           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3101           nat_affinity_init (vm);
3102         }
3103       else
3104         {
3105           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3106           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3107           sm->in2out_node_index = snat_in2out_node.index;
3108           sm->in2out_output_node_index = snat_in2out_output_node.index;
3109           sm->out2in_node_index = snat_out2in_node.index;
3110           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3111           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3112         }
3113       if (!static_mapping_only ||
3114           (static_mapping_only && static_mapping_connection_tracking))
3115         {
3116           /* *INDENT-OFF* */
3117           vec_foreach (tsm, sm->per_thread_data)
3118             {
3119               if (sm->endpoint_dependent)
3120                 {
3121                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3122                                          translation_buckets,
3123                                          translation_memory_size);
3124                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3125                                                       format_ed_session_kvp);
3126
3127                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3128                                          translation_buckets,
3129                                          translation_memory_size);
3130                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3131                                                       format_ed_session_kvp);
3132                 }
3133               else
3134                 {
3135                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3136                                         translation_buckets,
3137                                         translation_memory_size);
3138                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3139                                                      format_session_kvp);
3140
3141                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3142                                         translation_buckets,
3143                                         translation_memory_size);
3144                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3145                                                      format_session_kvp);
3146                 }
3147
3148               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3149                                     user_memory_size);
3150               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3151                                                  format_user_kvp);
3152             }
3153           /* *INDENT-ON* */
3154
3155         }
3156       else
3157         {
3158           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3159           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3160         }
3161       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3162                             "static_mapping_by_local", static_mapping_buckets,
3163                             static_mapping_memory_size);
3164       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3165                                          format_static_mapping_kvp);
3166
3167       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3168                             "static_mapping_by_external",
3169                             static_mapping_buckets,
3170                             static_mapping_memory_size);
3171       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3172                                          format_static_mapping_kvp);
3173     }
3174
3175   return 0;
3176 }
3177
3178 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3179
3180 static void
3181 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3182                                  uword opaque,
3183                                  u32 sw_if_index,
3184                                  ip4_address_t * address,
3185                                  u32 address_length,
3186                                  u32 if_address_index, u32 is_delete)
3187 {
3188   snat_main_t *sm = &snat_main;
3189   snat_static_map_resolve_t *rp;
3190   snat_static_mapping_t *m;
3191   snat_session_key_t m_key;
3192   clib_bihash_kv_8_8_t kv, value;
3193   int i, rv;
3194   ip4_address_t l_addr;
3195
3196   for (i = 0; i < vec_len (sm->to_resolve); i++)
3197     {
3198       rp = sm->to_resolve + i;
3199       if (rp->addr_only == 0)
3200         continue;
3201       if (rp->sw_if_index == sw_if_index)
3202         goto match;
3203     }
3204
3205   return;
3206
3207 match:
3208   m_key.addr.as_u32 = address->as_u32;
3209   m_key.port = rp->addr_only ? 0 : rp->e_port;
3210   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3211   m_key.fib_index = sm->outside_fib_index;
3212   kv.key = m_key.as_u64;
3213   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3214     m = 0;
3215   else
3216     m = pool_elt_at_index (sm->static_mappings, value.value);
3217
3218   if (!is_delete)
3219     {
3220       /* Don't trip over lease renewal, static config */
3221       if (m)
3222         return;
3223     }
3224   else
3225     {
3226       if (!m)
3227         return;
3228     }
3229
3230   /* Indetity mapping? */
3231   if (rp->l_addr.as_u32 == 0)
3232     l_addr.as_u32 = address[0].as_u32;
3233   else
3234     l_addr.as_u32 = rp->l_addr.as_u32;
3235   /* Add the static mapping */
3236   rv = snat_add_static_mapping (l_addr,
3237                                 address[0],
3238                                 rp->l_port,
3239                                 rp->e_port,
3240                                 rp->vrf_id,
3241                                 rp->addr_only, ~0 /* sw_if_index */ ,
3242                                 rp->proto, !is_delete, rp->twice_nat,
3243                                 rp->out2in_only, rp->tag, rp->identity_nat);
3244   if (rv)
3245     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3246 }
3247
3248 static void
3249 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3250                                        uword opaque,
3251                                        u32 sw_if_index,
3252                                        ip4_address_t * address,
3253                                        u32 address_length,
3254                                        u32 if_address_index, u32 is_delete)
3255 {
3256   snat_main_t *sm = &snat_main;
3257   snat_static_map_resolve_t *rp;
3258   ip4_address_t l_addr;
3259   int i, j;
3260   int rv;
3261   u8 twice_nat = 0;
3262   snat_address_t *addresses = sm->addresses;
3263
3264   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3265     {
3266       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3267         goto match;
3268     }
3269
3270   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3271     {
3272       twice_nat = 1;
3273       addresses = sm->twice_nat_addresses;
3274       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3275         goto match;
3276     }
3277
3278   return;
3279
3280 match:
3281   if (!is_delete)
3282     {
3283       /* Don't trip over lease renewal, static config */
3284       for (j = 0; j < vec_len (addresses); j++)
3285         if (addresses[j].addr.as_u32 == address->as_u32)
3286           return;
3287
3288       (void) snat_add_address (sm, address, ~0, twice_nat);
3289       /* Scan static map resolution vector */
3290       for (j = 0; j < vec_len (sm->to_resolve); j++)
3291         {
3292           rp = sm->to_resolve + j;
3293           if (rp->addr_only)
3294             continue;
3295           /* On this interface? */
3296           if (rp->sw_if_index == sw_if_index)
3297             {
3298               /* Indetity mapping? */
3299               if (rp->l_addr.as_u32 == 0)
3300                 l_addr.as_u32 = address[0].as_u32;
3301               else
3302                 l_addr.as_u32 = rp->l_addr.as_u32;
3303               /* Add the static mapping */
3304               rv = snat_add_static_mapping (l_addr,
3305                                             address[0],
3306                                             rp->l_port,
3307                                             rp->e_port,
3308                                             rp->vrf_id,
3309                                             rp->addr_only,
3310                                             ~0 /* sw_if_index */ ,
3311                                             rp->proto,
3312                                             rp->is_add, rp->twice_nat,
3313                                             rp->out2in_only, rp->tag,
3314                                             rp->identity_nat);
3315               if (rv)
3316                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3317             }
3318         }
3319       return;
3320     }
3321   else
3322     {
3323       (void) snat_del_address (sm, address[0], 1, twice_nat);
3324       return;
3325     }
3326 }
3327
3328
3329 int
3330 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3331                             u8 twice_nat)
3332 {
3333   ip4_main_t *ip4_main = sm->ip4_main;
3334   ip4_address_t *first_int_addr;
3335   snat_static_map_resolve_t *rp;
3336   u32 *indices_to_delete = 0;
3337   int i, j;
3338   u32 *auto_add_sw_if_indices =
3339     twice_nat ? sm->
3340     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3341
3342   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3343     );
3344
3345   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3346     {
3347       if (auto_add_sw_if_indices[i] == sw_if_index)
3348         {
3349           if (is_del)
3350             {
3351               /* if have address remove it */
3352               if (first_int_addr)
3353                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3354               else
3355                 {
3356                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3357                     {
3358                       rp = sm->to_resolve + j;
3359                       if (rp->sw_if_index == sw_if_index)
3360                         vec_add1 (indices_to_delete, j);
3361                     }
3362                   if (vec_len (indices_to_delete))
3363                     {
3364                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3365                         vec_del1 (sm->to_resolve, j);
3366                       vec_free (indices_to_delete);
3367                     }
3368                 }
3369               if (twice_nat)
3370                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3371               else
3372                 vec_del1 (sm->auto_add_sw_if_indices, i);
3373             }
3374           else
3375             return VNET_API_ERROR_VALUE_EXIST;
3376
3377           return 0;
3378         }
3379     }
3380
3381   if (is_del)
3382     return VNET_API_ERROR_NO_SUCH_ENTRY;
3383
3384   /* add to the auto-address list */
3385   if (twice_nat)
3386     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3387   else
3388     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3389
3390   /* If the address is already bound - or static - add it now */
3391   if (first_int_addr)
3392     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3393
3394   return 0;
3395 }
3396
3397 int
3398 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3399                    snat_protocol_t proto, u32 vrf_id, int is_in)
3400 {
3401   snat_main_per_thread_data_t *tsm;
3402   clib_bihash_kv_8_8_t kv, value;
3403   ip4_header_t ip;
3404   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3405   snat_session_key_t key;
3406   snat_session_t *s;
3407   clib_bihash_8_8_t *t;
3408
3409   if (sm->endpoint_dependent)
3410     return VNET_API_ERROR_UNSUPPORTED;
3411
3412   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3413   if (sm->num_workers > 1)
3414     tsm =
3415       vec_elt_at_index (sm->per_thread_data,
3416                         sm->worker_in2out_cb (&ip, fib_index));
3417   else
3418     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3419
3420   key.addr.as_u32 = addr->as_u32;
3421   key.port = clib_host_to_net_u16 (port);
3422   key.protocol = proto;
3423   key.fib_index = fib_index;
3424   kv.key = key.as_u64;
3425   t = is_in ? &tsm->in2out : &tsm->out2in;
3426   if (!clib_bihash_search_8_8 (t, &kv, &value))
3427     {
3428       if (pool_is_free_index (tsm->sessions, value.value))
3429         return VNET_API_ERROR_UNSPECIFIED;
3430
3431       s = pool_elt_at_index (tsm->sessions, value.value);
3432       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3433       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3434       return 0;
3435     }
3436
3437   return VNET_API_ERROR_NO_SUCH_ENTRY;
3438 }
3439
3440 int
3441 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3442                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3443                       u32 vrf_id, int is_in)
3444 {
3445   ip4_header_t ip;
3446   clib_bihash_16_8_t *t;
3447   nat_ed_ses_key_t key;
3448   clib_bihash_kv_16_8_t kv, value;
3449   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3450   snat_session_t *s;
3451   snat_main_per_thread_data_t *tsm;
3452
3453   if (!sm->endpoint_dependent)
3454     return VNET_API_ERROR_FEATURE_DISABLED;
3455
3456   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3457   if (sm->num_workers > 1)
3458     tsm =
3459       vec_elt_at_index (sm->per_thread_data,
3460                         sm->worker_in2out_cb (&ip, fib_index));
3461   else
3462     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3463
3464   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3465   key.l_addr.as_u32 = addr->as_u32;
3466   key.r_addr.as_u32 = eh_addr->as_u32;
3467   key.l_port = clib_host_to_net_u16 (port);
3468   key.r_port = clib_host_to_net_u16 (eh_port);
3469   key.proto = proto;
3470   key.fib_index = fib_index;
3471   kv.key[0] = key.as_u64[0];
3472   kv.key[1] = key.as_u64[1];
3473   if (clib_bihash_search_16_8 (t, &kv, &value))
3474     return VNET_API_ERROR_NO_SUCH_ENTRY;
3475
3476   if (pool_is_free_index (tsm->sessions, value.value))
3477     return VNET_API_ERROR_UNSPECIFIED;
3478   s = pool_elt_at_index (tsm->sessions, value.value);
3479   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3480   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3481   return 0;
3482 }
3483
3484 void
3485 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3486 {
3487   snat_main_t *sm = &snat_main;
3488
3489   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3490   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3491   sm->psid = psid;
3492   sm->psid_offset = psid_offset;
3493   sm->psid_length = psid_length;
3494 }
3495
3496 void
3497 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3498 {
3499   snat_main_t *sm = &snat_main;
3500
3501   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3502   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3503   sm->start_port = start_port;
3504   sm->end_port = end_port;
3505 }
3506
3507 void
3508 nat_set_alloc_addr_and_port_default (void)
3509 {
3510   snat_main_t *sm = &snat_main;
3511
3512   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3513   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3514 }
3515
3516 /*
3517  * fd.io coding-style-patch-verification: ON
3518  *
3519  * Local Variables:
3520  * eval: (c-set-style "gnu")
3521  * End:
3522  */