src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35
  36 #include <vpp/app/version.h>
  37
  38 snat_main_t snat_main;
  39
  40 /* *INDENT-OFF* */
  41
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat44-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  47 };
  48 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat44-out2in",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-dhcp-client-detect"),
  53 };
  54 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-classify",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-in2out",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-out2in",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-det-classify",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  74 };
  75 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-ed-in2out",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-ed-out2in",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  84                                "ip4-dhcp-client-detect"),
  85 };
  86 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-ed-classify",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-in2out-worker-handoff",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  95 };
  96 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-out2in-worker-handoff",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-handoff-classify",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-in2out-fast",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 111 };
 112 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-out2in-fast",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 116                                "ip4-dhcp-client-detect"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-hairpin-dst",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ed-hairpin-dst",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 127 };
 128
 129 /* Hook up output features */
 130 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 131   .arc_name = "ip4-output",
 132   .node_name = "nat44-in2out-output",
 133   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 136   .arc_name = "ip4-output",
 137   .node_name = "nat44-in2out-output-worker-handoff",
 138   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 141   .arc_name = "ip4-output",
 142   .node_name = "nat44-hairpin-src",
 143   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 144 };
 145 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-ed-in2out-output",
 148   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 149 };
 150 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-ed-hairpin-src",
 153   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 154 };
 155
 156 /* Hook up ip4-local features */
 157 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 158 {
 159   .arc_name = "ip4-local",
 160   .node_name = "nat44-hairpinning",
 161   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 164 {
 165   .arc_name = "ip4-local",
 166   .node_name = "nat44-ed-hairpinning",
 167   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 168 };
 169
 170
 171 VLIB_PLUGIN_REGISTER () = {
 172     .version = VPP_BUILD_VER,
 173     .description = "Network Address Translation",
 174 };
 175 /* *INDENT-ON* */
 176
 177 void
 178 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 179 {
 180   snat_session_key_t key;
 181   clib_bihash_kv_8_8_t kv;
 182   nat_ed_ses_key_t ed_key;
 183   clib_bihash_kv_16_8_t ed_kv;
 184   snat_main_per_thread_data_t *tsm =
 185     vec_elt_at_index (sm->per_thread_data, thread_index);
 186
 187   if (is_fwd_bypass_session (s))
 188     {
 189       ed_key.l_addr = s->in2out.addr;
 190       ed_key.r_addr = s->ext_host_addr;
 191       ed_key.l_port = s->in2out.port;
 192       ed_key.r_port = s->ext_host_port;
 193       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 194       ed_key.fib_index = 0;
 195       ed_kv.key[0] = ed_key.as_u64[0];
 196       ed_kv.key[1] = ed_key.as_u64[1];
 197       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 198         nat_log_warn ("in2out_ed key del failed");
 199       return;
 200     }
 201
 202   /* session lookup tables */
 203   if (is_ed_session (s))
 204     {
 205       if (is_affinity_sessions (s))
 206         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 207                              s->in2out.protocol, s->out2in.port);
 208       ed_key.l_addr = s->out2in.addr;
 209       ed_key.r_addr = s->ext_host_addr;
 210       ed_key.fib_index = s->out2in.fib_index;
 211       if (snat_is_unk_proto_session (s))
 212         {
 213           ed_key.proto = s->in2out.port;
 214           ed_key.r_port = 0;
 215           ed_key.l_port = 0;
 216         }
 217       else
 218         {
 219           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 220           ed_key.l_port = s->out2in.port;
 221           ed_key.r_port = s->ext_host_port;
 222         }
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 226         nat_log_warn ("out2in_ed key del failed");
 227       ed_key.l_addr = s->in2out.addr;
 228       ed_key.fib_index = s->in2out.fib_index;
 229       if (!snat_is_unk_proto_session (s))
 230         ed_key.l_port = s->in2out.port;
 231       if (is_twice_nat_session (s))
 232         {
 233           ed_key.r_addr = s->ext_host_nat_addr;
 234           ed_key.r_port = s->ext_host_nat_port;
 235         }
 236       ed_kv.key[0] = ed_key.as_u64[0];
 237       ed_kv.key[1] = ed_key.as_u64[1];
 238       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 239         nat_log_warn ("in2out_ed key del failed");
 240
 241       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 242                              &s->in2out.addr, s->in2out.port,
 243                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 244                              &s->out2in.addr, s->out2in.port,
 245                              &s->ext_host_addr, s->ext_host_port,
 246                              s->in2out.protocol, is_twice_nat_session (s));
 247     }
 248   else
 249     {
 250       kv.key = s->in2out.as_u64;
 251       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 252         nat_log_warn ("in2out key del failed");
 253       kv.key = s->out2in.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 255         nat_log_warn ("out2in key del failed");
 256
 257       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 258                                &s->in2out.addr, s->in2out.port,
 259                                &s->out2in.addr, s->out2in.port,
 260                                s->in2out.protocol);
 261     }
 262
 263   if (snat_is_unk_proto_session (s))
 264     return;
 265
 266   /* log NAT event */
 267   snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 268                                        s->out2in.addr.as_u32,
 269                                        s->in2out.protocol,
 270                                        s->in2out.port,
 271                                        s->out2in.port, s->in2out.fib_index);
 272
 273   /* Twice NAT address and port for external host */
 274   if (is_twice_nat_session (s))
 275     {
 276       key.protocol = s->in2out.protocol;
 277       key.port = s->ext_host_nat_port;
 278       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 279       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 280                                           thread_index, &key);
 281     }
 282
 283   if (snat_is_session_static (s))
 284     return;
 285
 286   snat_free_outside_address_and_port (sm->addresses, thread_index,
 287                                       &s->out2in);
 288 }
 289
 290 snat_user_t *
 291 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 292                         u32 thread_index)
 293 {
 294   snat_user_t *u = 0;
 295   snat_user_key_t user_key;
 296   clib_bihash_kv_8_8_t kv, value;
 297   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 298   dlist_elt_t *per_user_list_head_elt;
 299
 300   user_key.addr.as_u32 = addr->as_u32;
 301   user_key.fib_index = fib_index;
 302   kv.key = user_key.as_u64;
 303
 304   /* Ever heard of the "user" = src ip4 address before? */
 305   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 306     {
 307       /* no, make a new one */
 308       pool_get (tsm->users, u);
 309       clib_memset (u, 0, sizeof (*u));
 310       u->addr.as_u32 = addr->as_u32;
 311       u->fib_index = fib_index;
 312
 313       pool_get (tsm->list_pool, per_user_list_head_elt);
 314
 315       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 316         tsm->list_pool;
 317
 318       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 319
 320       kv.value = u - tsm->users;
 321
 322       /* add user */
 323       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 324         nat_log_warn ("user_hash keay add failed");
 325
 326       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 327                                pool_elts (tsm->users));
 328     }
 329   else
 330     {
 331       u = pool_elt_at_index (tsm->users, value.value);
 332     }
 333
 334   return u;
 335 }
 336
 337 snat_session_t *
 338 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 339                               u32 thread_index)
 340 {
 341   snat_session_t *s;
 342   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 343   u32 oldest_per_user_translation_list_index, session_index;
 344   dlist_elt_t *oldest_per_user_translation_list_elt;
 345   dlist_elt_t *per_user_translation_list_elt;
 346
 347   /* Over quota? Recycle the least recently used translation */
 348   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 349     {
 350       oldest_per_user_translation_list_index =
 351         clib_dlist_remove_head (tsm->list_pool,
 352                                 u->sessions_per_user_list_head_index);
 353
 354       ASSERT (oldest_per_user_translation_list_index != ~0);
 355
 356       /* Add it back to the end of the LRU list */
 357       clib_dlist_addtail (tsm->list_pool,
 358                           u->sessions_per_user_list_head_index,
 359                           oldest_per_user_translation_list_index);
 360       /* Get the list element */
 361       oldest_per_user_translation_list_elt =
 362         pool_elt_at_index (tsm->list_pool,
 363                            oldest_per_user_translation_list_index);
 364
 365       /* Get the session index from the list element */
 366       session_index = oldest_per_user_translation_list_elt->value;
 367
 368       /* Get the session */
 369       s = pool_elt_at_index (tsm->sessions, session_index);
 370       nat_free_session_data (sm, s, thread_index);
 371       if (snat_is_session_static (s))
 372         u->nstaticsessions--;
 373       else
 374         u->nsessions--;
 375       s->flags = 0;
 376       s->total_bytes = 0;
 377       s->total_pkts = 0;
 378       s->state = 0;
 379       s->ext_host_addr.as_u32 = 0;
 380       s->ext_host_port = 0;
 381       s->ext_host_nat_addr.as_u32 = 0;
 382       s->ext_host_nat_port = 0;
 383     }
 384   else
 385     {
 386       pool_get (tsm->sessions, s);
 387       clib_memset (s, 0, sizeof (*s));
 388
 389       /* Create list elts */
 390       pool_get (tsm->list_pool, per_user_translation_list_elt);
 391       clib_dlist_init (tsm->list_pool,
 392                        per_user_translation_list_elt - tsm->list_pool);
 393
 394       per_user_translation_list_elt->value = s - tsm->sessions;
 395       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 396       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 397
 398       clib_dlist_addtail (tsm->list_pool,
 399                           s->per_user_list_head_index,
 400                           per_user_translation_list_elt - tsm->list_pool);
 401
 402       s->user_index = u - tsm->users;
 403       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 404                                pool_elts (tsm->sessions));
 405     }
 406
 407   return s;
 408 }
 409
 410 snat_session_t *
 411 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 412                       f64 now)
 413 {
 414   snat_session_t *s;
 415   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 416   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 417   u32 oldest_index;
 418   u64 sess_timeout_time;
 419
 420   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 421     goto alloc_new;
 422
 423   oldest_index =
 424     clib_dlist_remove_head (tsm->list_pool,
 425                             u->sessions_per_user_list_head_index);
 426   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 427   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 428   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 429   if (now >= sess_timeout_time)
 430     {
 431       clib_dlist_addtail (tsm->list_pool,
 432                           u->sessions_per_user_list_head_index, oldest_index);
 433       nat_free_session_data (sm, s, thread_index);
 434       if (snat_is_session_static (s))
 435         u->nstaticsessions--;
 436       else
 437         u->nsessions--;
 438       s->flags = 0;
 439       s->total_bytes = 0;
 440       s->total_pkts = 0;
 441       s->state = 0;
 442       s->ext_host_addr.as_u32 = 0;
 443       s->ext_host_port = 0;
 444       s->ext_host_nat_addr.as_u32 = 0;
 445       s->ext_host_nat_port = 0;
 446     }
 447   else
 448     {
 449       clib_dlist_addhead (tsm->list_pool,
 450                           u->sessions_per_user_list_head_index, oldest_index);
 451       if ((u->nsessions + u->nstaticsessions) >=
 452           sm->max_translations_per_user)
 453         {
 454           nat_log_warn ("max translations per user %U", format_ip4_address,
 455                         &u->addr);
 456           snat_ipfix_logging_max_entries_per_user
 457             (sm->max_translations_per_user, u->addr.as_u32);
 458           return 0;
 459         }
 460       else
 461         {
 462         alloc_new:
 463           pool_get (tsm->sessions, s);
 464           clib_memset (s, 0, sizeof (*s));
 465
 466           /* Create list elts */
 467           pool_get (tsm->list_pool, per_user_translation_list_elt);
 468           clib_dlist_init (tsm->list_pool,
 469                            per_user_translation_list_elt - tsm->list_pool);
 470
 471           per_user_translation_list_elt->value = s - tsm->sessions;
 472           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 473           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 474
 475           clib_dlist_addtail (tsm->list_pool,
 476                               s->per_user_list_head_index,
 477                               per_user_translation_list_elt - tsm->list_pool);
 478         }
 479
 480       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 481                                pool_elts (tsm->sessions));
 482     }
 483
 484   return s;
 485 }
 486
 487 void
 488 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 489                           int is_add)
 490 {
 491   fib_prefix_t prefix = {
 492     .fp_len = p_len,
 493     .fp_proto = FIB_PROTOCOL_IP4,
 494     .fp_addr = {
 495                 .ip4.as_u32 = addr->as_u32,
 496                 },
 497   };
 498   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 499
 500   if (is_add)
 501     fib_table_entry_update_one_path (fib_index,
 502                                      &prefix,
 503                                      FIB_SOURCE_PLUGIN_LOW,
 504                                      (FIB_ENTRY_FLAG_CONNECTED |
 505                                       FIB_ENTRY_FLAG_LOCAL |
 506                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 507                                      DPO_PROTO_IP4,
 508                                      NULL,
 509                                      sw_if_index,
 510                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 511   else
 512     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 513 }
 514
 515 int
 516 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 517                   u8 twice_nat)
 518 {
 519   snat_address_t *ap;
 520   snat_interface_t *i;
 521   vlib_thread_main_t *tm = vlib_get_thread_main ();
 522
 523   if (twice_nat && !sm->endpoint_dependent)
 524     return VNET_API_ERROR_FEATURE_DISABLED;
 525
 526   /* Check if address already exists */
 527   /* *INDENT-OFF* */
 528   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 529     {
 530       if (ap->addr.as_u32 == addr->as_u32)
 531         return VNET_API_ERROR_VALUE_EXIST;
 532     }
 533   /* *INDENT-ON* */
 534
 535   if (twice_nat)
 536     vec_add2 (sm->twice_nat_addresses, ap, 1);
 537   else
 538     vec_add2 (sm->addresses, ap, 1);
 539
 540   ap->addr = *addr;
 541   if (vrf_id != ~0)
 542     ap->fib_index =
 543       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 544                                          FIB_SOURCE_PLUGIN_LOW);
 545   else
 546     ap->fib_index = ~0;
 547 #define _(N, i, n, s) \
 548   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 549   ap->busy_##n##_ports = 0; \
 550   ap->busy_##n##_ports_per_thread = 0;\
 551   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 552   foreach_snat_protocol
 553 #undef _
 554     if (twice_nat)
 555     return 0;
 556
 557   /* Add external address to FIB */
 558   /* *INDENT-OFF* */
 559   pool_foreach (i, sm->interfaces,
 560   ({
 561     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 562       continue;
 563
 564     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 565     break;
 566   }));
 567   pool_foreach (i, sm->output_feature_interfaces,
 568   ({
 569     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 570       continue;
 571
 572     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 573     break;
 574   }));
 575   /* *INDENT-ON* */
 576
 577   return 0;
 578 }
 579
 580 static int
 581 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 582 {
 583   snat_static_mapping_t *m;
 584   /* *INDENT-OFF* */
 585   pool_foreach (m, sm->static_mappings,
 586   ({
 587       if (is_addr_only_static_mapping (m) ||
 588           is_out2in_only_static_mapping (m) ||
 589           is_identity_static_mapping (m))
 590         continue;
 591       if (m->external_addr.as_u32 == addr.as_u32)
 592         return 1;
 593   }));
 594   /* *INDENT-ON* */
 595
 596   return 0;
 597 }
 598
 599 void
 600 increment_v4_address (ip4_address_t * a)
 601 {
 602   u32 v;
 603
 604   v = clib_net_to_host_u32 (a->as_u32) + 1;
 605   a->as_u32 = clib_host_to_net_u32 (v);
 606 }
 607
 608 static void
 609 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 610                                        ip4_address_t l_addr,
 611                                        u16 l_port,
 612                                        u32 sw_if_index,
 613                                        u16 e_port,
 614                                        u32 vrf_id,
 615                                        snat_protocol_t proto,
 616                                        int addr_only, int is_add, u8 * tag,
 617                                        int twice_nat, int out2in_only,
 618                                        int identity_nat)
 619 {
 620   snat_static_map_resolve_t *rp;
 621
 622   vec_add2 (sm->to_resolve, rp, 1);
 623   rp->l_addr.as_u32 = l_addr.as_u32;
 624   rp->l_port = l_port;
 625   rp->sw_if_index = sw_if_index;
 626   rp->e_port = e_port;
 627   rp->vrf_id = vrf_id;
 628   rp->proto = proto;
 629   rp->addr_only = addr_only;
 630   rp->is_add = is_add;
 631   rp->twice_nat = twice_nat;
 632   rp->out2in_only = out2in_only;
 633   rp->identity_nat = identity_nat;
 634   rp->tag = vec_dup (tag);
 635 }
 636
 637 static u32
 638 get_thread_idx_by_port (u16 e_port)
 639 {
 640   snat_main_t *sm = &snat_main;
 641   u32 thread_idx = sm->num_workers;
 642   if (sm->num_workers > 1)
 643     {
 644       thread_idx =
 645         sm->first_worker_index +
 646         sm->workers[(e_port - 1024) / sm->port_per_thread];
 647     }
 648   return thread_idx;
 649 }
 650
 651 int
 652 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 653                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 654                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 655                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 656                          u8 identity_nat)
 657 {
 658   snat_main_t *sm = &snat_main;
 659   snat_static_mapping_t *m;
 660   snat_session_key_t m_key;
 661   clib_bihash_kv_8_8_t kv, value;
 662   snat_address_t *a = 0;
 663   u32 fib_index = ~0;
 664   snat_interface_t *interface;
 665   int i;
 666   snat_main_per_thread_data_t *tsm;
 667   snat_user_key_t u_key;
 668   snat_user_t *u;
 669   dlist_elt_t *head, *elt;
 670   u32 elt_index, head_index;
 671   u32 ses_index;
 672   u64 user_index;
 673   snat_session_t *s;
 674   snat_static_map_resolve_t *rp, *rp_match = 0;
 675   nat44_lb_addr_port_t *local;
 676   u32 find = ~0;
 677
 678   if (!sm->endpoint_dependent)
 679     {
 680       if (twice_nat || out2in_only)
 681         return VNET_API_ERROR_FEATURE_DISABLED;
 682     }
 683
 684   /* If the external address is a specific interface address */
 685   if (sw_if_index != ~0)
 686     {
 687       ip4_address_t *first_int_addr;
 688
 689       for (i = 0; i < vec_len (sm->to_resolve); i++)
 690         {
 691           rp = sm->to_resolve + i;
 692           if (rp->sw_if_index != sw_if_index ||
 693               rp->l_addr.as_u32 != l_addr.as_u32 ||
 694               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 695             continue;
 696
 697           if (!addr_only)
 698             {
 699               if (rp->l_port != l_port || rp->e_port != e_port
 700                   || rp->proto != proto)
 701                 continue;
 702             }
 703
 704           rp_match = rp;
 705           break;
 706         }
 707
 708       /* Might be already set... */
 709       first_int_addr = ip4_interface_first_address
 710         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 711
 712       if (is_add)
 713         {
 714           if (rp_match)
 715             return VNET_API_ERROR_VALUE_EXIST;
 716
 717           snat_add_static_mapping_when_resolved
 718             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 719              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 720
 721           /* DHCP resolution required? */
 722           if (first_int_addr == 0)
 723             {
 724               return 0;
 725             }
 726           else
 727             {
 728               e_addr.as_u32 = first_int_addr->as_u32;
 729               /* Identity mapping? */
 730               if (l_addr.as_u32 == 0)
 731                 l_addr.as_u32 = e_addr.as_u32;
 732             }
 733         }
 734       else
 735         {
 736           if (!rp_match)
 737             return VNET_API_ERROR_NO_SUCH_ENTRY;
 738
 739           vec_del1 (sm->to_resolve, i);
 740
 741           if (first_int_addr)
 742             {
 743               e_addr.as_u32 = first_int_addr->as_u32;
 744               /* Identity mapping? */
 745               if (l_addr.as_u32 == 0)
 746                 l_addr.as_u32 = e_addr.as_u32;
 747             }
 748           else
 749             return 0;
 750         }
 751     }
 752
 753   m_key.addr = e_addr;
 754   m_key.port = addr_only ? 0 : e_port;
 755   m_key.protocol = addr_only ? 0 : proto;
 756   m_key.fib_index = 0;
 757   kv.key = m_key.as_u64;
 758   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 759     m = 0;
 760   else
 761     m = pool_elt_at_index (sm->static_mappings, value.value);
 762
 763   if (is_add)
 764     {
 765       if (m)
 766         {
 767           if (is_identity_static_mapping (m))
 768             {
 769               /* *INDENT-OFF* */
 770               pool_foreach (local, m->locals,
 771               ({
 772                 if (local->vrf_id == vrf_id)
 773                   return VNET_API_ERROR_VALUE_EXIST;
 774               }));
 775               /* *INDENT-ON* */
 776               pool_get (m->locals, local);
 777               local->vrf_id = vrf_id;
 778               local->fib_index =
 779                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 780                                                    FIB_SOURCE_PLUGIN_LOW);
 781               m_key.addr = m->local_addr;
 782               m_key.port = m->local_port;
 783               m_key.protocol = m->proto;
 784               m_key.fib_index = local->fib_index;
 785               kv.key = m_key.as_u64;
 786               kv.value = m - sm->static_mappings;
 787               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 788               return 0;
 789             }
 790           else
 791             return VNET_API_ERROR_VALUE_EXIST;
 792         }
 793
 794       if (twice_nat && addr_only)
 795         return VNET_API_ERROR_UNSUPPORTED;
 796
 797       /* Convert VRF id to FIB index */
 798       if (vrf_id != ~0)
 799         fib_index =
 800           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 801                                              FIB_SOURCE_PLUGIN_LOW);
 802       /* If not specified use inside VRF id from SNAT plugin startup config */
 803       else
 804         {
 805           fib_index = sm->inside_fib_index;
 806           vrf_id = sm->inside_vrf_id;
 807         }
 808
 809       if (!(out2in_only || identity_nat))
 810         {
 811           m_key.addr = l_addr;
 812           m_key.port = addr_only ? 0 : l_port;
 813           m_key.protocol = addr_only ? 0 : proto;
 814           m_key.fib_index = fib_index;
 815           kv.key = m_key.as_u64;
 816           if (!clib_bihash_search_8_8
 817               (&sm->static_mapping_by_local, &kv, &value))
 818             return VNET_API_ERROR_VALUE_EXIST;
 819         }
 820
 821       /* Find external address in allocated addresses and reserve port for
 822          address and port pair mapping when dynamic translations enabled */
 823       if (!(addr_only || sm->static_mapping_only || out2in_only))
 824         {
 825           for (i = 0; i < vec_len (sm->addresses); i++)
 826             {
 827               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 828                 {
 829                   a = sm->addresses + i;
 830                   /* External port must be unused */
 831                   switch (proto)
 832                     {
 833 #define _(N, j, n, s) \
 834                     case SNAT_PROTOCOL_##N: \
 835                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 836                         return VNET_API_ERROR_INVALID_VALUE; \
 837                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 838                       if (e_port > 1024) \
 839                         { \
 840                           a->busy_##n##_ports++; \
 841                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 842                         } \
 843                       break;
 844                       foreach_snat_protocol
 845 #undef _
 846                     default:
 847                       nat_log_info ("unknown protocol");
 848                       return VNET_API_ERROR_INVALID_VALUE_2;
 849                     }
 850                   break;
 851                 }
 852             }
 853           /* External address must be allocated */
 854           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 855             {
 856               if (sw_if_index != ~0)
 857                 {
 858                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 859                     {
 860                       rp = sm->to_resolve + i;
 861                       if (rp->addr_only)
 862                         continue;
 863                       if (rp->sw_if_index != sw_if_index &&
 864                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 865                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 866                           rp->e_port != e_port && rp->proto != proto)
 867                         continue;
 868
 869                       vec_del1 (sm->to_resolve, i);
 870                       break;
 871                     }
 872                 }
 873               return VNET_API_ERROR_NO_SUCH_ENTRY;
 874             }
 875         }
 876
 877       pool_get (sm->static_mappings, m);
 878       clib_memset (m, 0, sizeof (*m));
 879       m->tag = vec_dup (tag);
 880       m->local_addr = l_addr;
 881       m->external_addr = e_addr;
 882       m->twice_nat = twice_nat;
 883       if (out2in_only)
 884         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 885       if (addr_only)
 886         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 887       if (identity_nat)
 888         {
 889           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 890           pool_get (m->locals, local);
 891           local->vrf_id = vrf_id;
 892           local->fib_index = fib_index;
 893         }
 894       else
 895         {
 896           m->vrf_id = vrf_id;
 897           m->fib_index = fib_index;
 898         }
 899       if (!addr_only)
 900         {
 901           m->local_port = l_port;
 902           m->external_port = e_port;
 903           m->proto = proto;
 904         }
 905
 906       if (sm->num_workers > 1)
 907         {
 908           ip4_header_t ip = {
 909             .src_address = m->local_addr,
 910           };
 911           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 912           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 913         }
 914       else
 915         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 916
 917       m_key.addr = m->local_addr;
 918       m_key.port = m->local_port;
 919       m_key.protocol = m->proto;
 920       m_key.fib_index = fib_index;
 921       kv.key = m_key.as_u64;
 922       kv.value = m - sm->static_mappings;
 923       if (!out2in_only)
 924         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 925
 926       m_key.addr = m->external_addr;
 927       m_key.port = m->external_port;
 928       m_key.fib_index = 0;
 929       kv.key = m_key.as_u64;
 930       kv.value = m - sm->static_mappings;
 931       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 932
 933       /* Delete dynamic sessions matching local address (+ local port) */
 934       if (!(sm->static_mapping_only))
 935         {
 936           u_key.addr = m->local_addr;
 937           u_key.fib_index = m->fib_index;
 938           kv.key = u_key.as_u64;
 939           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 940             {
 941               user_index = value.value;
 942               u = pool_elt_at_index (tsm->users, user_index);
 943               if (u->nsessions)
 944                 {
 945                   head_index = u->sessions_per_user_list_head_index;
 946                   head = pool_elt_at_index (tsm->list_pool, head_index);
 947                   elt_index = head->next;
 948                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 949                   ses_index = elt->value;
 950                   while (ses_index != ~0)
 951                     {
 952                       s = pool_elt_at_index (tsm->sessions, ses_index);
 953                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 954                       ses_index = elt->value;
 955
 956                       if (snat_is_session_static (s))
 957                         continue;
 958
 959                       if (!addr_only
 960                           && (clib_net_to_host_u16 (s->in2out.port) !=
 961                               m->local_port))
 962                         continue;
 963
 964                       nat_free_session_data (sm, s,
 965                                              tsm - sm->per_thread_data);
 966                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 967
 968                       if (!addr_only && !sm->endpoint_dependent)
 969                         break;
 970                     }
 971                 }
 972             }
 973         }
 974     }
 975   else
 976     {
 977       if (!m)
 978         {
 979           if (sw_if_index != ~0)
 980             return 0;
 981           else
 982             return VNET_API_ERROR_NO_SUCH_ENTRY;
 983         }
 984
 985       if (identity_nat)
 986         {
 987           if (vrf_id == ~0)
 988             vrf_id = sm->inside_vrf_id;
 989
 990           /* *INDENT-OFF* */
 991           pool_foreach (local, m->locals,
 992           ({
 993             if (local->vrf_id == vrf_id)
 994               find = local - m->locals;
 995           }));
 996           /* *INDENT-ON* */
 997           if (find == ~0)
 998             return VNET_API_ERROR_NO_SUCH_ENTRY;
 999
1000           local = pool_elt_at_index (m->locals, find);
1001           fib_index = local->fib_index;
1002           pool_put (m->locals, local);
1003         }
1004       else
1005         fib_index = m->fib_index;
1006
1007       /* Free external address port */
1008       if (!(addr_only || sm->static_mapping_only || out2in_only))
1009         {
1010           for (i = 0; i < vec_len (sm->addresses); i++)
1011             {
1012               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1013                 {
1014                   a = sm->addresses + i;
1015                   switch (proto)
1016                     {
1017 #define _(N, j, n, s) \
1018                     case SNAT_PROTOCOL_##N: \
1019                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1020                       if (e_port > 1024) \
1021                         { \
1022                           a->busy_##n##_ports--; \
1023                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1024                         } \
1025                       break;
1026                       foreach_snat_protocol
1027 #undef _
1028                     default:
1029                       nat_log_info ("unknown protocol");
1030                       return VNET_API_ERROR_INVALID_VALUE_2;
1031                     }
1032                   break;
1033                 }
1034             }
1035         }
1036
1037       if (sm->num_workers > 1)
1038         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1039       else
1040         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1041
1042       m_key.addr = m->local_addr;
1043       m_key.port = m->local_port;
1044       m_key.protocol = m->proto;
1045       m_key.fib_index = fib_index;
1046       kv.key = m_key.as_u64;
1047       if (!out2in_only)
1048         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1049
1050       /* Delete session(s) for static mapping if exist */
1051       if (!(sm->static_mapping_only) ||
1052           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1053         {
1054           u_key.addr = m->local_addr;
1055           u_key.fib_index = fib_index;
1056           kv.key = u_key.as_u64;
1057           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1058             {
1059               user_index = value.value;
1060               u = pool_elt_at_index (tsm->users, user_index);
1061               if (u->nstaticsessions)
1062                 {
1063                   head_index = u->sessions_per_user_list_head_index;
1064                   head = pool_elt_at_index (tsm->list_pool, head_index);
1065                   elt_index = head->next;
1066                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1067                   ses_index = elt->value;
1068                   while (ses_index != ~0)
1069                     {
1070                       s = pool_elt_at_index (tsm->sessions, ses_index);
1071                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1072                       ses_index = elt->value;
1073
1074                       if (!addr_only)
1075                         {
1076                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1077                               (clib_net_to_host_u16 (s->out2in.port) !=
1078                                e_port))
1079                             continue;
1080                         }
1081
1082                       if (is_lb_session (s))
1083                         continue;
1084
1085                       if (!snat_is_session_static (s))
1086                         continue;
1087
1088                       nat_free_session_data (sm, s,
1089                                              tsm - sm->per_thread_data);
1090                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1091
1092                       if (!addr_only && !sm->endpoint_dependent)
1093                         break;
1094                     }
1095                 }
1096             }
1097         }
1098
1099       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1100       if (pool_elts (m->locals))
1101         return 0;
1102
1103       m_key.addr = m->external_addr;
1104       m_key.port = m->external_port;
1105       m_key.fib_index = 0;
1106       kv.key = m_key.as_u64;
1107       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1108
1109       vec_free (m->tag);
1110       vec_free (m->workers);
1111       /* Delete static mapping from pool */
1112       pool_put (sm->static_mappings, m);
1113     }
1114
1115   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1116     return 0;
1117
1118   /* Add/delete external address to FIB */
1119   /* *INDENT-OFF* */
1120   pool_foreach (interface, sm->interfaces,
1121   ({
1122     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1123       continue;
1124
1125     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1126     break;
1127   }));
1128   pool_foreach (interface, sm->output_feature_interfaces,
1129   ({
1130     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1131       continue;
1132
1133     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1134     break;
1135   }));
1136   /* *INDENT-ON* */
1137
1138   return 0;
1139 }
1140
1141 int
1142 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1143                                  snat_protocol_t proto,
1144                                  nat44_lb_addr_port_t * locals, u8 is_add,
1145                                  twice_nat_type_t twice_nat, u8 out2in_only,
1146                                  u8 * tag, u32 affinity)
1147 {
1148   snat_main_t *sm = &snat_main;
1149   snat_static_mapping_t *m;
1150   snat_session_key_t m_key;
1151   clib_bihash_kv_8_8_t kv, value;
1152   snat_address_t *a = 0;
1153   int i;
1154   nat44_lb_addr_port_t *local;
1155   u32 elt_index, head_index, ses_index;
1156   snat_main_per_thread_data_t *tsm;
1157   snat_user_key_t u_key;
1158   snat_user_t *u;
1159   snat_session_t *s;
1160   dlist_elt_t *head, *elt;
1161   uword *bitmap = 0;
1162
1163   if (!sm->endpoint_dependent)
1164     return VNET_API_ERROR_FEATURE_DISABLED;
1165
1166   m_key.addr = e_addr;
1167   m_key.port = e_port;
1168   m_key.protocol = proto;
1169   m_key.fib_index = 0;
1170   kv.key = m_key.as_u64;
1171   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1172     m = 0;
1173   else
1174     m = pool_elt_at_index (sm->static_mappings, value.value);
1175
1176   if (is_add)
1177     {
1178       if (m)
1179         return VNET_API_ERROR_VALUE_EXIST;
1180
1181       if (vec_len (locals) < 2)
1182         return VNET_API_ERROR_INVALID_VALUE;
1183
1184       /* Find external address in allocated addresses and reserve port for
1185          address and port pair mapping when dynamic translations enabled */
1186       if (!(sm->static_mapping_only || out2in_only))
1187         {
1188           for (i = 0; i < vec_len (sm->addresses); i++)
1189             {
1190               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1191                 {
1192                   a = sm->addresses + i;
1193                   /* External port must be unused */
1194                   switch (proto)
1195                     {
1196 #define _(N, j, n, s) \
1197                     case SNAT_PROTOCOL_##N: \
1198                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1199                         return VNET_API_ERROR_INVALID_VALUE; \
1200                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1201                       if (e_port > 1024) \
1202                         { \
1203                           a->busy_##n##_ports++; \
1204                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1205                         } \
1206                       break;
1207                       foreach_snat_protocol
1208 #undef _
1209                     default:
1210                       nat_log_info ("unknown protocol");
1211                       return VNET_API_ERROR_INVALID_VALUE_2;
1212                     }
1213                   break;
1214                 }
1215             }
1216           /* External address must be allocated */
1217           if (!a)
1218             return VNET_API_ERROR_NO_SUCH_ENTRY;
1219         }
1220
1221       pool_get (sm->static_mappings, m);
1222       clib_memset (m, 0, sizeof (*m));
1223       m->tag = vec_dup (tag);
1224       m->external_addr = e_addr;
1225       m->external_port = e_port;
1226       m->proto = proto;
1227       m->twice_nat = twice_nat;
1228       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1229       if (out2in_only)
1230         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1231       m->affinity = affinity;
1232
1233       if (affinity)
1234         m->affinity_per_service_list_head_index =
1235           nat_affinity_get_per_service_list_head_index ();
1236       else
1237         m->affinity_per_service_list_head_index = ~0;
1238
1239       m_key.addr = m->external_addr;
1240       m_key.port = m->external_port;
1241       m_key.protocol = m->proto;
1242       m_key.fib_index = 0;
1243       kv.key = m_key.as_u64;
1244       kv.value = m - sm->static_mappings;
1245       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1246         {
1247           nat_log_err ("static_mapping_by_external key add failed");
1248           return VNET_API_ERROR_UNSPECIFIED;
1249         }
1250
1251       m_key.fib_index = m->fib_index;
1252       for (i = 0; i < vec_len (locals); i++)
1253         {
1254           locals[i].fib_index =
1255             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1256                                                locals[i].vrf_id,
1257                                                FIB_SOURCE_PLUGIN_LOW);
1258           m_key.addr = locals[i].addr;
1259           m_key.fib_index = locals[i].fib_index;
1260           if (!out2in_only)
1261             {
1262               m_key.port = locals[i].port;
1263               kv.key = m_key.as_u64;
1264               kv.value = m - sm->static_mappings;
1265               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1266             }
1267           locals[i].prefix = (i == 0) ? locals[i].probability :
1268             (locals[i - 1].prefix + locals[i].probability);
1269           pool_get (m->locals, local);
1270           *local = locals[i];
1271           if (sm->num_workers > 1)
1272             {
1273               ip4_header_t ip = {
1274                 .src_address = locals[i].addr,
1275               };
1276               bitmap =
1277                 clib_bitmap_set (bitmap,
1278                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1279             }
1280         }
1281
1282       /* Assign workers */
1283       if (sm->num_workers > 1)
1284         {
1285           /* *INDENT-OFF* */
1286           clib_bitmap_foreach (i, bitmap,
1287             ({
1288                vec_add1(m->workers, i);
1289             }));
1290           /* *INDENT-ON* */
1291         }
1292     }
1293   else
1294     {
1295       if (!m)
1296         return VNET_API_ERROR_NO_SUCH_ENTRY;
1297
1298       if (!is_lb_static_mapping (m))
1299         return VNET_API_ERROR_INVALID_VALUE;
1300
1301       /* Free external address port */
1302       if (!(sm->static_mapping_only || out2in_only))
1303         {
1304           for (i = 0; i < vec_len (sm->addresses); i++)
1305             {
1306               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1307                 {
1308                   a = sm->addresses + i;
1309                   switch (proto)
1310                     {
1311 #define _(N, j, n, s) \
1312                     case SNAT_PROTOCOL_##N: \
1313                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1314                       if (e_port > 1024) \
1315                         { \
1316                           a->busy_##n##_ports--; \
1317                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1318                         } \
1319                       break;
1320                       foreach_snat_protocol
1321 #undef _
1322                     default:
1323                       nat_log_info ("unknown protocol");
1324                       return VNET_API_ERROR_INVALID_VALUE_2;
1325                     }
1326                   break;
1327                 }
1328             }
1329         }
1330
1331       m_key.addr = m->external_addr;
1332       m_key.port = m->external_port;
1333       m_key.protocol = m->proto;
1334       m_key.fib_index = 0;
1335       kv.key = m_key.as_u64;
1336       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1337         {
1338           nat_log_err ("static_mapping_by_external key del failed");
1339           return VNET_API_ERROR_UNSPECIFIED;
1340         }
1341
1342       /* *INDENT-OFF* */
1343       pool_foreach (local, m->locals,
1344       ({
1345           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1346                             FIB_SOURCE_PLUGIN_LOW);
1347           m_key.addr = local->addr;
1348           if (!out2in_only)
1349             {
1350               m_key.port = local->port;
1351               m_key.fib_index = local->fib_index;
1352               kv.key = m_key.as_u64;
1353               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1354                 {
1355                   nat_log_err ("static_mapping_by_local key del failed");
1356                   return VNET_API_ERROR_UNSPECIFIED;
1357                 }
1358             }
1359
1360           if (sm->num_workers > 1)
1361             {
1362               ip4_header_t ip = {
1363                 .src_address = local->addr,
1364               };
1365               tsm = vec_elt_at_index (sm->per_thread_data,
1366                                       sm->worker_in2out_cb (&ip, m->fib_index));
1367             }
1368           else
1369             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1370
1371           /* Delete sessions */
1372           u_key.addr = local->addr;
1373           u_key.fib_index = local->fib_index;
1374           kv.key = u_key.as_u64;
1375           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1376             {
1377               u = pool_elt_at_index (tsm->users, value.value);
1378               if (u->nstaticsessions)
1379                 {
1380                   head_index = u->sessions_per_user_list_head_index;
1381                   head = pool_elt_at_index (tsm->list_pool, head_index);
1382                   elt_index = head->next;
1383                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1384                   ses_index = elt->value;
1385                   while (ses_index != ~0)
1386                     {
1387                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1388                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1389                       ses_index = elt->value;
1390
1391                       if (!(is_lb_session (s)))
1392                         continue;
1393
1394                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1395                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1396                         continue;
1397
1398                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1399                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1400                     }
1401                 }
1402             }
1403       }));
1404       /* *INDENT-ON* */
1405       if (m->affinity)
1406         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1407       pool_free (m->locals);
1408       vec_free (m->tag);
1409       vec_free (m->workers);
1410
1411       pool_put (sm->static_mappings, m);
1412     }
1413
1414   return 0;
1415 }
1416
1417 int
1418 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1419                                        ip4_address_t l_addr, u16 l_port,
1420                                        snat_protocol_t proto, u32 vrf_id,
1421                                        u8 probability, u8 is_add)
1422 {
1423   snat_main_t *sm = &snat_main;
1424   snat_static_mapping_t *m = 0;
1425   snat_session_key_t m_key;
1426   clib_bihash_kv_8_8_t kv, value;
1427   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1428   snat_main_per_thread_data_t *tsm;
1429   snat_user_key_t u_key;
1430   snat_user_t *u;
1431   snat_session_t *s;
1432   dlist_elt_t *head, *elt;
1433   u32 elt_index, head_index, ses_index, *locals = 0;
1434   uword *bitmap = 0;
1435   int i;
1436
1437   if (!sm->endpoint_dependent)
1438     return VNET_API_ERROR_FEATURE_DISABLED;
1439
1440   m_key.addr = e_addr;
1441   m_key.port = e_port;
1442   m_key.protocol = proto;
1443   m_key.fib_index = 0;
1444   kv.key = m_key.as_u64;
1445   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1446     m = pool_elt_at_index (sm->static_mappings, value.value);
1447
1448   if (!m)
1449     return VNET_API_ERROR_NO_SUCH_ENTRY;
1450
1451   if (!is_lb_static_mapping (m))
1452     return VNET_API_ERROR_INVALID_VALUE;
1453
1454   /* *INDENT-OFF* */
1455   pool_foreach (local, m->locals,
1456   ({
1457     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1458         (local->vrf_id == vrf_id))
1459       {
1460         match_local = local;
1461         break;
1462       }
1463   }));
1464   /* *INDENT-ON* */
1465
1466   if (is_add)
1467     {
1468       if (match_local)
1469         return VNET_API_ERROR_VALUE_EXIST;
1470
1471       pool_get (m->locals, local);
1472       clib_memset (local, 0, sizeof (*local));
1473       local->addr.as_u32 = l_addr.as_u32;
1474       local->port = l_port;
1475       local->probability = probability;
1476       local->vrf_id = vrf_id;
1477       local->fib_index =
1478         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1479                                            FIB_SOURCE_PLUGIN_LOW);
1480
1481       if (!is_out2in_only_static_mapping (m))
1482         {
1483           m_key.addr = l_addr;
1484           m_key.port = l_port;
1485           m_key.fib_index = local->fib_index;
1486           kv.key = m_key.as_u64;
1487           kv.value = m - sm->static_mappings;
1488           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1489             nat_log_err ("static_mapping_by_local key add failed");
1490         }
1491     }
1492   else
1493     {
1494       if (!match_local)
1495         return VNET_API_ERROR_NO_SUCH_ENTRY;
1496
1497       if (pool_elts (m->locals) < 3)
1498         return VNET_API_ERROR_UNSPECIFIED;
1499
1500       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1501                         FIB_SOURCE_PLUGIN_LOW);
1502
1503       if (!is_out2in_only_static_mapping (m))
1504         {
1505           m_key.addr = l_addr;
1506           m_key.port = l_port;
1507           m_key.fib_index = match_local->fib_index;
1508           kv.key = m_key.as_u64;
1509           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1510             nat_log_err ("static_mapping_by_local key del failed");
1511         }
1512
1513       if (sm->num_workers > 1)
1514         {
1515           ip4_header_t ip = {
1516             .src_address = local->addr,
1517           };
1518           tsm = vec_elt_at_index (sm->per_thread_data,
1519                                   sm->worker_in2out_cb (&ip, m->fib_index));
1520         }
1521       else
1522         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1523
1524       /* Delete sessions */
1525       u_key.addr = match_local->addr;
1526       u_key.fib_index = match_local->fib_index;
1527       kv.key = u_key.as_u64;
1528       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1529         {
1530           u = pool_elt_at_index (tsm->users, value.value);
1531           if (u->nstaticsessions)
1532             {
1533               head_index = u->sessions_per_user_list_head_index;
1534               head = pool_elt_at_index (tsm->list_pool, head_index);
1535               elt_index = head->next;
1536               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1537               ses_index = elt->value;
1538               while (ses_index != ~0)
1539                 {
1540                   s = pool_elt_at_index (tsm->sessions, ses_index);
1541                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1542                   ses_index = elt->value;
1543
1544                   if (!(is_lb_session (s)))
1545                     continue;
1546
1547                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1548                       (clib_net_to_host_u16 (s->in2out.port) !=
1549                        match_local->port))
1550                     continue;
1551
1552                   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1553                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1554                 }
1555             }
1556         }
1557
1558       pool_put (m->locals, match_local);
1559     }
1560
1561   vec_free (m->workers);
1562
1563   /* *INDENT-OFF* */
1564   pool_foreach (local, m->locals,
1565   ({
1566     vec_add1 (locals, local - m->locals);
1567     if (sm->num_workers > 1)
1568       {
1569         ip4_header_t ip;
1570         ip.src_address.as_u32 = local->addr.as_u32,
1571         bitmap = clib_bitmap_set (bitmap,
1572                                   sm->worker_in2out_cb (&ip, local->fib_index),
1573                                   1);
1574       }
1575   }));
1576   /* *INDENT-ON* */
1577
1578   ASSERT (vec_len (locals) > 1);
1579
1580   local = pool_elt_at_index (m->locals, locals[0]);
1581   local->prefix = local->probability;
1582   for (i = 1; i < vec_len (locals); i++)
1583     {
1584       local = pool_elt_at_index (m->locals, locals[i]);
1585       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1586       local->prefix = local->probability + prev_local->prefix;
1587     }
1588
1589   /* Assign workers */
1590   if (sm->num_workers > 1)
1591     {
1592       /* *INDENT-OFF* */
1593       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1594       /* *INDENT-ON* */
1595     }
1596
1597   return 0;
1598 }
1599
1600 int
1601 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1602                   u8 twice_nat)
1603 {
1604   snat_address_t *a = 0;
1605   snat_session_t *ses;
1606   u32 *ses_to_be_removed = 0, *ses_index;
1607   snat_main_per_thread_data_t *tsm;
1608   snat_static_mapping_t *m;
1609   snat_interface_t *interface;
1610   int i;
1611   snat_address_t *addresses =
1612     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1613
1614   /* Find SNAT address */
1615   for (i = 0; i < vec_len (addresses); i++)
1616     {
1617       if (addresses[i].addr.as_u32 == addr.as_u32)
1618         {
1619           a = addresses + i;
1620           break;
1621         }
1622     }
1623   if (!a)
1624     return VNET_API_ERROR_NO_SUCH_ENTRY;
1625
1626   if (delete_sm)
1627     {
1628       /* *INDENT-OFF* */
1629       pool_foreach (m, sm->static_mappings,
1630       ({
1631           if (m->external_addr.as_u32 == addr.as_u32)
1632             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1633                                             m->local_port, m->external_port,
1634                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1635                                             m->proto, 0, m->twice_nat,
1636                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1637       }));
1638       /* *INDENT-ON* */
1639     }
1640   else
1641     {
1642       /* Check if address is used in some static mapping */
1643       if (is_snat_address_used_in_static_mapping (sm, addr))
1644         {
1645           nat_log_notice ("address used in static mapping");
1646           return VNET_API_ERROR_UNSPECIFIED;
1647         }
1648     }
1649
1650   if (a->fib_index != ~0)
1651     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1652
1653   /* Delete sessions using address */
1654   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1655     {
1656       /* *INDENT-OFF* */
1657       vec_foreach (tsm, sm->per_thread_data)
1658         {
1659           pool_foreach (ses, tsm->sessions, ({
1660             if (ses->out2in.addr.as_u32 == addr.as_u32)
1661               {
1662                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1663                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1664               }
1665           }));
1666
1667           vec_foreach (ses_index, ses_to_be_removed)
1668             {
1669               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1670               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1671             }
1672
1673           vec_free (ses_to_be_removed);
1674         }
1675       /* *INDENT-ON* */
1676     }
1677
1678 #define _(N, i, n, s) \
1679   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1680   vec_free (a->busy_##n##_ports_per_thread);
1681   foreach_snat_protocol
1682 #undef _
1683     if (twice_nat)
1684     {
1685       vec_del1 (sm->twice_nat_addresses, i);
1686       return 0;
1687     }
1688   else
1689     vec_del1 (sm->addresses, i);
1690
1691   /* Delete external address from FIB */
1692   /* *INDENT-OFF* */
1693   pool_foreach (interface, sm->interfaces,
1694   ({
1695     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1696       continue;
1697
1698     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1699     break;
1700   }));
1701   pool_foreach (interface, sm->output_feature_interfaces,
1702   ({
1703     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1704       continue;
1705
1706     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1707     break;
1708   }));
1709   /* *INDENT-ON* */
1710
1711   return 0;
1712 }
1713
1714 int
1715 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1716 {
1717   snat_main_t *sm = &snat_main;
1718   snat_interface_t *i;
1719   const char *feature_name, *del_feature_name;
1720   snat_address_t *ap;
1721   snat_static_mapping_t *m;
1722   snat_det_map_t *dm;
1723   nat_outside_fib_t *outside_fib;
1724   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1725                                                        sw_if_index);
1726
1727   if (sm->out2in_dpo && !is_inside)
1728     return VNET_API_ERROR_UNSUPPORTED;
1729
1730   /* *INDENT-OFF* */
1731   pool_foreach (i, sm->output_feature_interfaces,
1732   ({
1733     if (i->sw_if_index == sw_if_index)
1734       return VNET_API_ERROR_VALUE_EXIST;
1735   }));
1736   /* *INDENT-ON* */
1737
1738   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1739     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1740   else
1741     {
1742       if (sm->num_workers > 1 && !sm->deterministic)
1743         feature_name =
1744           is_inside ? "nat44-in2out-worker-handoff" :
1745           "nat44-out2in-worker-handoff";
1746       else if (sm->deterministic)
1747         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1748       else if (sm->endpoint_dependent)
1749         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1750       else
1751         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1752     }
1753
1754   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1755     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1756                                                       NAT_FQ_NELTS);
1757
1758   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1759     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1760                                                       NAT_FQ_NELTS);
1761
1762   if (!is_inside)
1763     {
1764       /* *INDENT-OFF* */
1765       vec_foreach (outside_fib, sm->outside_fibs)
1766         {
1767           if (outside_fib->fib_index == fib_index)
1768             {
1769               if (is_del)
1770                 {
1771                   outside_fib->refcount--;
1772                   if (!outside_fib->refcount)
1773                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1774                 }
1775               else
1776                 outside_fib->refcount++;
1777               goto feature_set;
1778             }
1779         }
1780       /* *INDENT-ON* */
1781       if (!is_del)
1782         {
1783           vec_add2 (sm->outside_fibs, outside_fib, 1);
1784           outside_fib->refcount = 1;
1785           outside_fib->fib_index = fib_index;
1786         }
1787     }
1788 feature_set:
1789   /* *INDENT-OFF* */
1790   pool_foreach (i, sm->interfaces,
1791   ({
1792     if (i->sw_if_index == sw_if_index)
1793       {
1794         if (is_del)
1795           {
1796             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1797               {
1798                 if (is_inside)
1799                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1800                 else
1801                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1802
1803                 if (sm->num_workers > 1 && !sm->deterministic)
1804                   {
1805                     del_feature_name = "nat44-handoff-classify";
1806                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1807                                                  "nat44-out2in-worker-handoff";
1808                   }
1809                 else if (sm->deterministic)
1810                   {
1811                     del_feature_name = "nat44-det-classify";
1812                     feature_name = !is_inside ?  "nat44-det-in2out" :
1813                                                  "nat44-det-out2in";
1814                   }
1815                 else if (sm->endpoint_dependent)
1816                   {
1817                     del_feature_name = "nat44-ed-classify";
1818                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1819                                                  "nat44-ed-out2in";
1820                   }
1821                 else
1822                   {
1823                     del_feature_name = "nat44-classify";
1824                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1825                   }
1826
1827                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1828                                              sw_if_index, 0, 0, 0);
1829                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1830                                              sw_if_index, 1, 0, 0);
1831                 if (!is_inside)
1832                   {
1833                     if (sm->endpoint_dependent)
1834                       vnet_feature_enable_disable ("ip4-local",
1835                                                    "nat44-ed-hairpinning",
1836                                                    sw_if_index, 1, 0, 0);
1837                     else if (!sm->deterministic)
1838                       vnet_feature_enable_disable ("ip4-local",
1839                                                    "nat44-hairpinning",
1840                                                    sw_if_index, 1, 0, 0);
1841                   }
1842               }
1843             else
1844               {
1845                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1846                                              sw_if_index, 0, 0, 0);
1847                 pool_put (sm->interfaces, i);
1848                 if (is_inside)
1849                   {
1850                     if (sm->endpoint_dependent)
1851                       vnet_feature_enable_disable ("ip4-local",
1852                                                    "nat44-ed-hairpinning",
1853                                                    sw_if_index, 0, 0, 0);
1854                     else if (!sm->deterministic)
1855                       vnet_feature_enable_disable ("ip4-local",
1856                                                    "nat44-hairpinning",
1857                                                    sw_if_index, 0, 0, 0);
1858                   }
1859               }
1860           }
1861         else
1862           {
1863             if ((nat_interface_is_inside(i) && is_inside) ||
1864                 (nat_interface_is_outside(i) && !is_inside))
1865               return 0;
1866
1867             if (sm->num_workers > 1 && !sm->deterministic)
1868               {
1869                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1870                                                  "nat44-out2in-worker-handoff";
1871                 feature_name = "nat44-handoff-classify";
1872               }
1873             else if (sm->deterministic)
1874               {
1875                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1876                                                  "nat44-det-out2in";
1877                 feature_name = "nat44-det-classify";
1878               }
1879             else if (sm->endpoint_dependent)
1880               {
1881                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1882                                                  "nat44-ed-out2in";
1883                 feature_name = "nat44-ed-classify";
1884               }
1885             else
1886               {
1887                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1888                 feature_name = "nat44-classify";
1889               }
1890
1891             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1892                                          sw_if_index, 0, 0, 0);
1893             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1894                                          sw_if_index, 1, 0, 0);
1895             if (!is_inside)
1896               {
1897                 if (sm->endpoint_dependent)
1898                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1899                                                sw_if_index, 0, 0, 0);
1900                 else if (!sm->deterministic)
1901                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1902                                                sw_if_index, 0, 0, 0);
1903               }
1904             goto set_flags;
1905           }
1906
1907         goto fib;
1908       }
1909   }));
1910   /* *INDENT-ON* */
1911
1912   if (is_del)
1913     return VNET_API_ERROR_NO_SUCH_ENTRY;
1914
1915   pool_get (sm->interfaces, i);
1916   i->sw_if_index = sw_if_index;
1917   i->flags = 0;
1918   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1919                                0);
1920
1921   if (is_inside && !sm->out2in_dpo)
1922     {
1923       if (sm->endpoint_dependent)
1924         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1925                                      sw_if_index, 1, 0, 0);
1926       else if (!sm->deterministic)
1927         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1928                                      sw_if_index, 1, 0, 0);
1929     }
1930
1931 set_flags:
1932   if (is_inside)
1933     {
1934       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1935       return 0;
1936     }
1937   else
1938     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1939
1940   /* Add/delete external addresses to FIB */
1941 fib:
1942   /* *INDENT-OFF* */
1943   vec_foreach (ap, sm->addresses)
1944     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1945
1946   pool_foreach (m, sm->static_mappings,
1947   ({
1948     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1949       continue;
1950
1951     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1952   }));
1953
1954   pool_foreach (dm, sm->det_maps,
1955   ({
1956     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1957   }));
1958   /* *INDENT-ON* */
1959
1960   return 0;
1961 }
1962
1963 int
1964 snat_interface_add_del_output_feature (u32 sw_if_index,
1965                                        u8 is_inside, int is_del)
1966 {
1967   snat_main_t *sm = &snat_main;
1968   snat_interface_t *i;
1969   snat_address_t *ap;
1970   snat_static_mapping_t *m;
1971
1972   if (sm->deterministic ||
1973       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1974     return VNET_API_ERROR_UNSUPPORTED;
1975
1976   /* *INDENT-OFF* */
1977   pool_foreach (i, sm->interfaces,
1978   ({
1979     if (i->sw_if_index == sw_if_index)
1980       return VNET_API_ERROR_VALUE_EXIST;
1981   }));
1982   /* *INDENT-ON* */
1983
1984   if (is_inside)
1985     {
1986       if (sm->endpoint_dependent)
1987         {
1988           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
1989                                        sw_if_index, !is_del, 0, 0);
1990           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
1991                                        sw_if_index, !is_del, 0, 0);
1992         }
1993       else
1994         {
1995           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1996                                        sw_if_index, !is_del, 0, 0);
1997           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1998                                        sw_if_index, !is_del, 0, 0);
1999         }
2000       goto fq;
2001     }
2002
2003   if (sm->num_workers > 1)
2004     {
2005       vnet_feature_enable_disable ("ip4-unicast",
2006                                    "nat44-out2in-worker-handoff",
2007                                    sw_if_index, !is_del, 0, 0);
2008       vnet_feature_enable_disable ("ip4-output",
2009                                    "nat44-in2out-output-worker-handoff",
2010                                    sw_if_index, !is_del, 0, 0);
2011     }
2012   else
2013     {
2014       if (sm->endpoint_dependent)
2015         {
2016           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2017                                        sw_if_index, !is_del, 0, 0);
2018           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2019                                        sw_if_index, !is_del, 0, 0);
2020         }
2021       else
2022         {
2023           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2024                                        sw_if_index, !is_del, 0, 0);
2025           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2026                                        sw_if_index, !is_del, 0, 0);
2027         }
2028     }
2029
2030 fq:
2031   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2032     sm->fq_in2out_output_index =
2033       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2034
2035   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2036     sm->fq_out2in_index =
2037       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2038
2039   /* *INDENT-OFF* */
2040   pool_foreach (i, sm->output_feature_interfaces,
2041   ({
2042     if (i->sw_if_index == sw_if_index)
2043       {
2044         if (is_del)
2045           pool_put (sm->output_feature_interfaces, i);
2046         else
2047           return VNET_API_ERROR_VALUE_EXIST;
2048
2049         goto fib;
2050       }
2051   }));
2052   /* *INDENT-ON* */
2053
2054   if (is_del)
2055     return VNET_API_ERROR_NO_SUCH_ENTRY;
2056
2057   pool_get (sm->output_feature_interfaces, i);
2058   i->sw_if_index = sw_if_index;
2059   i->flags = 0;
2060   if (is_inside)
2061     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2062   else
2063     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2064
2065   /* Add/delete external addresses to FIB */
2066 fib:
2067   if (is_inside)
2068     return 0;
2069
2070   /* *INDENT-OFF* */
2071   vec_foreach (ap, sm->addresses)
2072     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2073
2074   pool_foreach (m, sm->static_mappings,
2075   ({
2076     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2077       continue;
2078
2079     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2080   }));
2081   /* *INDENT-ON* */
2082
2083   return 0;
2084 }
2085
2086 int
2087 snat_set_workers (uword * bitmap)
2088 {
2089   snat_main_t *sm = &snat_main;
2090   int i, j = 0;
2091
2092   if (sm->num_workers < 2)
2093     return VNET_API_ERROR_FEATURE_DISABLED;
2094
2095   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2096     return VNET_API_ERROR_INVALID_WORKER;
2097
2098   vec_free (sm->workers);
2099   /* *INDENT-OFF* */
2100   clib_bitmap_foreach (i, bitmap,
2101     ({
2102       vec_add1(sm->workers, i);
2103       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2104       j++;
2105     }));
2106   /* *INDENT-ON* */
2107
2108   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2109   sm->num_snat_thread = _vec_len (sm->workers);
2110
2111   return 0;
2112 }
2113
2114
2115 static void
2116 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2117                                        uword opaque,
2118                                        u32 sw_if_index,
2119                                        ip4_address_t * address,
2120                                        u32 address_length,
2121                                        u32 if_address_index, u32 is_delete);
2122
2123 static void
2124 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2125                                  uword opaque,
2126                                  u32 sw_if_index,
2127                                  ip4_address_t * address,
2128                                  u32 address_length,
2129                                  u32 if_address_index, u32 is_delete);
2130
2131 static int
2132 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2133                                  u32 fib_index,
2134                                  u32 thread_index,
2135                                  snat_session_key_t * k,
2136                                  u16 port_per_thread, u32 snat_thread_index);
2137
2138 static clib_error_t *
2139 snat_init (vlib_main_t * vm)
2140 {
2141   snat_main_t *sm = &snat_main;
2142   clib_error_t *error = 0;
2143   ip4_main_t *im = &ip4_main;
2144   ip_lookup_main_t *lm = &im->lookup_main;
2145   uword *p;
2146   vlib_thread_registration_t *tr;
2147   vlib_thread_main_t *tm = vlib_get_thread_main ();
2148   uword *bitmap = 0;
2149   u32 i;
2150   ip4_add_del_interface_address_callback_t cb4;
2151   vlib_node_t *error_drop_node;
2152
2153   sm->vlib_main = vm;
2154   sm->vnet_main = vnet_get_main ();
2155   sm->ip4_main = im;
2156   sm->ip4_lookup_main = lm;
2157   sm->api_main = &api_main;
2158   sm->first_worker_index = 0;
2159   sm->num_workers = 0;
2160   sm->num_snat_thread = 1;
2161   sm->workers = 0;
2162   sm->port_per_thread = 0xffff - 1024;
2163   sm->fq_in2out_index = ~0;
2164   sm->fq_out2in_index = ~0;
2165   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2166   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2167   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2168   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2169   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2170   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2171   sm->forwarding_enabled = 0;
2172   sm->log_class = vlib_log_register_class ("nat", 0);
2173   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2174   sm->error_node_index = error_drop_node->index;
2175   sm->mss_clamping = 0;
2176
2177   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2178   if (p)
2179     {
2180       tr = (vlib_thread_registration_t *) p[0];
2181       if (tr)
2182         {
2183           sm->num_workers = tr->count;
2184           sm->first_worker_index = tr->first_index;
2185         }
2186     }
2187
2188   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2189
2190   /* Use all available workers by default */
2191   if (sm->num_workers > 1)
2192     {
2193       for (i = 0; i < sm->num_workers; i++)
2194         bitmap = clib_bitmap_set (bitmap, i, 1);
2195       snat_set_workers (bitmap);
2196       clib_bitmap_free (bitmap);
2197     }
2198   else
2199     {
2200       sm->per_thread_data[0].snat_thread_index = 0;
2201     }
2202
2203   error = snat_api_init (vm, sm);
2204   if (error)
2205     return error;
2206
2207   /* Set up the interface address add/del callback */
2208   cb4.function = snat_ip4_add_del_interface_address_cb;
2209   cb4.function_opaque = 0;
2210
2211   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2212
2213   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2214   cb4.function_opaque = 0;
2215
2216   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2217
2218   nat_dpo_module_init ();
2219
2220   /* Init counters */
2221   sm->total_users.name = "total-users";
2222   sm->total_users.stat_segment_name = "/nat44/total-users";
2223   vlib_validate_simple_counter (&sm->total_users, 0);
2224   vlib_zero_simple_counter (&sm->total_users, 0);
2225   sm->total_sessions.name = "total-sessions";
2226   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2227   vlib_validate_simple_counter (&sm->total_sessions, 0);
2228   vlib_zero_simple_counter (&sm->total_sessions, 0);
2229
2230   /* Init IPFIX logging */
2231   snat_ipfix_logging_init (vm);
2232
2233   /* Init NAT64 */
2234   error = nat64_init (vm);
2235   if (error)
2236     return error;
2237
2238   dslite_init (vm);
2239
2240   nat66_init ();
2241
2242   /* Init virtual fragmenentation reassembly */
2243   return nat_reass_init (vm);
2244 }
2245
2246 VLIB_INIT_FUNCTION (snat_init);
2247
2248 void
2249 snat_free_outside_address_and_port (snat_address_t * addresses,
2250                                     u32 thread_index, snat_session_key_t * k)
2251 {
2252   snat_address_t *a;
2253   u32 address_index;
2254   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2255
2256   for (address_index = 0; address_index < vec_len (addresses);
2257        address_index++)
2258     {
2259       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2260         break;
2261     }
2262
2263   ASSERT (address_index < vec_len (addresses));
2264
2265   a = addresses + address_index;
2266
2267   switch (k->protocol)
2268     {
2269 #define _(N, i, n, s) \
2270     case SNAT_PROTOCOL_##N: \
2271       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2272         port_host_byte_order) == 1); \
2273       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2274         port_host_byte_order, 0); \
2275       a->busy_##n##_ports--; \
2276       a->busy_##n##_ports_per_thread[thread_index]--; \
2277       break;
2278       foreach_snat_protocol
2279 #undef _
2280     default:
2281       nat_log_info ("unknown protocol");
2282       return;
2283     }
2284 }
2285
2286 int
2287 snat_static_mapping_match (snat_main_t * sm,
2288                            snat_session_key_t match,
2289                            snat_session_key_t * mapping,
2290                            u8 by_external,
2291                            u8 * is_addr_only,
2292                            twice_nat_type_t * twice_nat,
2293                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2294                            u8 * is_identity_nat)
2295 {
2296   clib_bihash_kv_8_8_t kv, value;
2297   snat_static_mapping_t *m;
2298   snat_session_key_t m_key;
2299   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2300   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2301   u8 backend_index;
2302   nat44_lb_addr_port_t *local;
2303
2304   m_key.fib_index = match.fib_index;
2305   if (by_external)
2306     {
2307       mapping_hash = &sm->static_mapping_by_external;
2308       m_key.fib_index = 0;
2309     }
2310
2311   m_key.addr = match.addr;
2312   m_key.port = clib_net_to_host_u16 (match.port);
2313   m_key.protocol = match.protocol;
2314
2315   kv.key = m_key.as_u64;
2316
2317   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2318     {
2319       /* Try address only mapping */
2320       m_key.port = 0;
2321       m_key.protocol = 0;
2322       kv.key = m_key.as_u64;
2323       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2324         return 1;
2325     }
2326
2327   m = pool_elt_at_index (sm->static_mappings, value.value);
2328
2329   if (by_external)
2330     {
2331       if (is_lb_static_mapping (m))
2332         {
2333           if (PREDICT_FALSE (lb != 0))
2334             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2335           if (m->affinity)
2336             {
2337               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2338                                               match.protocol, match.port,
2339                                               &backend_index))
2340                 goto get_local;
2341
2342               local = pool_elt_at_index (m->locals, backend_index);
2343               mapping->addr = local->addr;
2344               mapping->port = clib_host_to_net_u16 (local->port);
2345               mapping->fib_index = local->fib_index;
2346               goto end;
2347             }
2348         get_local:
2349           /* *INDENT-OFF* */
2350           pool_foreach_index (i, m->locals,
2351           ({
2352             vec_add1 (tmp, i);
2353           }));
2354           /* *INDENT-ON* */
2355           hi = vec_len (tmp) - 1;
2356           local = pool_elt_at_index (m->locals, tmp[hi]);
2357           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2358           while (lo < hi)
2359             {
2360               mid = ((hi - lo) >> 1) + lo;
2361               local = pool_elt_at_index (m->locals, tmp[mid]);
2362               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2363             }
2364           local = pool_elt_at_index (m->locals, tmp[lo]);
2365           if (!(local->prefix >= rand))
2366             return 1;
2367           if (PREDICT_FALSE (sm->num_workers > 1))
2368             {
2369               ip4_header_t ip = {
2370                 .src_address = local->addr,
2371               };
2372               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2373                   vlib_get_thread_index ())
2374                 goto get_local;
2375             }
2376           mapping->addr = local->addr;
2377           mapping->port = clib_host_to_net_u16 (local->port);
2378           mapping->fib_index = local->fib_index;
2379           if (m->affinity)
2380             {
2381               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2382                                                 match.protocol, match.port,
2383                                                 tmp[lo], m->affinity,
2384                                                 m->affinity_per_service_list_head_index))
2385                 nat_log_info ("create affinity record failed");
2386             }
2387           vec_free (tmp);
2388         }
2389       else
2390         {
2391           if (PREDICT_FALSE (lb != 0))
2392             *lb = NO_LB_NAT;
2393           mapping->fib_index = m->fib_index;
2394           mapping->addr = m->local_addr;
2395           /* Address only mapping doesn't change port */
2396           mapping->port = is_addr_only_static_mapping (m) ? match.port
2397             : clib_host_to_net_u16 (m->local_port);
2398         }
2399       mapping->protocol = m->proto;
2400     }
2401   else
2402     {
2403       mapping->addr = m->external_addr;
2404       /* Address only mapping doesn't change port */
2405       mapping->port = is_addr_only_static_mapping (m) ? match.port
2406         : clib_host_to_net_u16 (m->external_port);
2407       mapping->fib_index = sm->outside_fib_index;
2408     }
2409
2410 end:
2411   if (PREDICT_FALSE (is_addr_only != 0))
2412     *is_addr_only = is_addr_only_static_mapping (m);
2413
2414   if (PREDICT_FALSE (twice_nat != 0))
2415     *twice_nat = m->twice_nat;
2416
2417   if (PREDICT_FALSE (is_identity_nat != 0))
2418     *is_identity_nat = is_identity_static_mapping (m);
2419
2420   return 0;
2421 }
2422
2423 static_always_inline u16
2424 snat_random_port (u16 min, u16 max)
2425 {
2426   snat_main_t *sm = &snat_main;
2427   return min + random_u32 (&sm->random_seed) /
2428     (random_u32_max () / (max - min + 1) + 1);
2429 }
2430
2431 int
2432 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2433                                      u32 fib_index,
2434                                      u32 thread_index,
2435                                      snat_session_key_t * k,
2436                                      u16 port_per_thread,
2437                                      u32 snat_thread_index)
2438 {
2439   snat_main_t *sm = &snat_main;
2440
2441   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2442                                   port_per_thread, snat_thread_index);
2443 }
2444
2445 static int
2446 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2447                                  u32 fib_index,
2448                                  u32 thread_index,
2449                                  snat_session_key_t * k,
2450                                  u16 port_per_thread, u32 snat_thread_index)
2451 {
2452   int i;
2453   snat_address_t *a, *ga = 0;
2454   u32 portnum;
2455
2456   for (i = 0; i < vec_len (addresses); i++)
2457     {
2458       a = addresses + i;
2459       switch (k->protocol)
2460         {
2461 #define _(N, j, n, s) \
2462         case SNAT_PROTOCOL_##N: \
2463           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2464             { \
2465               if (a->fib_index == fib_index) \
2466                 { \
2467                   while (1) \
2468                     { \
2469                       portnum = (port_per_thread * \
2470                         snat_thread_index) + \
2471                         snat_random_port(1, port_per_thread) + 1024; \
2472                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2473                         continue; \
2474                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2475                       a->busy_##n##_ports_per_thread[thread_index]++; \
2476                       a->busy_##n##_ports++; \
2477                       k->addr = a->addr; \
2478                       k->port = clib_host_to_net_u16(portnum); \
2479                       return 0; \
2480                     } \
2481                 } \
2482               else if (a->fib_index == ~0) \
2483                 { \
2484                   ga = a; \
2485                 } \
2486             } \
2487           break;
2488           foreach_snat_protocol
2489 #undef _
2490         default:
2491           nat_log_info ("unknown protocol");
2492           return 1;
2493         }
2494
2495     }
2496
2497   if (ga)
2498     {
2499       a = ga;
2500       switch (k->protocol)
2501         {
2502 #define _(N, j, n, s) \
2503         case SNAT_PROTOCOL_##N: \
2504           while (1) \
2505             { \
2506               portnum = (port_per_thread * \
2507                 snat_thread_index) + \
2508                 snat_random_port(1, port_per_thread) + 1024; \
2509               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2510                 continue; \
2511               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2512               a->busy_##n##_ports_per_thread[thread_index]++; \
2513               a->busy_##n##_ports++; \
2514               k->addr = a->addr; \
2515               k->port = clib_host_to_net_u16(portnum); \
2516               return 0; \
2517             }
2518           break;
2519           foreach_snat_protocol
2520 #undef _
2521         default:
2522           nat_log_info ("unknown protocol");
2523           return 1;
2524         }
2525     }
2526
2527   /* Totally out of translations to use... */
2528   snat_ipfix_logging_addresses_exhausted (0);
2529   return 1;
2530 }
2531
2532 static int
2533 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2534                               u32 fib_index,
2535                               u32 thread_index,
2536                               snat_session_key_t * k,
2537                               u16 port_per_thread, u32 snat_thread_index)
2538 {
2539   snat_main_t *sm = &snat_main;
2540   snat_address_t *a = addresses;
2541   u16 m, ports, portnum, A, j;
2542   m = 16 - (sm->psid_offset + sm->psid_length);
2543   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2544
2545   if (!vec_len (addresses))
2546     goto exhausted;
2547
2548   switch (k->protocol)
2549     {
2550 #define _(N, i, n, s) \
2551     case SNAT_PROTOCOL_##N: \
2552       if (a->busy_##n##_ports < ports) \
2553         { \
2554           while (1) \
2555             { \
2556               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2557               j = snat_random_port(0, pow2_mask(m)); \
2558               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2559               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2560                 continue; \
2561               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2562               a->busy_##n##_ports++; \
2563               k->addr = a->addr; \
2564               k->port = clib_host_to_net_u16 (portnum); \
2565               return 0; \
2566             } \
2567         } \
2568       break;
2569       foreach_snat_protocol
2570 #undef _
2571     default:
2572       nat_log_info ("unknown protocol");
2573       return 1;
2574     }
2575
2576 exhausted:
2577   /* Totally out of translations to use... */
2578   snat_ipfix_logging_addresses_exhausted (0);
2579   return 1;
2580 }
2581
2582 static int
2583 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2584                                u32 fib_index,
2585                                u32 thread_index,
2586                                snat_session_key_t * k,
2587                                u16 port_per_thread, u32 snat_thread_index)
2588 {
2589   snat_main_t *sm = &snat_main;
2590   snat_address_t *a = addresses;
2591   u16 portnum, ports;
2592
2593   ports = sm->end_port - sm->start_port + 1;
2594
2595   if (!vec_len (addresses))
2596     goto exhausted;
2597
2598   switch (k->protocol)
2599     {
2600 #define _(N, i, n, s) \
2601     case SNAT_PROTOCOL_##N: \
2602       if (a->busy_##n##_ports < ports) \
2603         { \
2604           while (1) \
2605             { \
2606               portnum = snat_random_port(sm->start_port, sm->end_port); \
2607               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2608                 continue; \
2609               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2610               a->busy_##n##_ports++; \
2611               k->addr = a->addr; \
2612               k->port = clib_host_to_net_u16 (portnum); \
2613               return 0; \
2614             } \
2615         } \
2616       break;
2617       foreach_snat_protocol
2618 #undef _
2619     default:
2620       nat_log_info ("unknown protocol");
2621       return 1;
2622     }
2623
2624 exhausted:
2625   /* Totally out of translations to use... */
2626   snat_ipfix_logging_addresses_exhausted (0);
2627   return 1;
2628 }
2629
2630 void
2631 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2632 {
2633   dpo_id_t dpo_v4 = DPO_INVALID;
2634   fib_prefix_t pfx = {
2635     .fp_proto = FIB_PROTOCOL_IP4,
2636     .fp_len = 32,
2637     .fp_addr.ip4.as_u32 = addr.as_u32,
2638   };
2639
2640   if (is_add)
2641     {
2642       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2643       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2644                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2645       dpo_reset (&dpo_v4);
2646     }
2647   else
2648     {
2649       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2650     }
2651 }
2652
2653 u8 *
2654 format_session_kvp (u8 * s, va_list * args)
2655 {
2656   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2657   snat_session_key_t k;
2658
2659   k.as_u64 = v->key;
2660
2661   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2662
2663   return s;
2664 }
2665
2666 u8 *
2667 format_static_mapping_kvp (u8 * s, va_list * args)
2668 {
2669   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2670   snat_session_key_t k;
2671
2672   k.as_u64 = v->key;
2673
2674   s = format (s, "%U static-mapping-index %llu",
2675               format_static_mapping_key, &k, v->value);
2676
2677   return s;
2678 }
2679
2680 u8 *
2681 format_user_kvp (u8 * s, va_list * args)
2682 {
2683   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2684   snat_user_key_t k;
2685
2686   k.as_u64 = v->key;
2687
2688   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2689               k.fib_index, v->value);
2690
2691   return s;
2692 }
2693
2694 u8 *
2695 format_ed_session_kvp (u8 * s, va_list * args)
2696 {
2697   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2698   nat_ed_ses_key_t k;
2699
2700   k.as_u64[0] = v->key[0];
2701   k.as_u64[1] = v->key[1];
2702
2703   s =
2704     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2705             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2706             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2707             format_ip_protocol, k.proto, k.fib_index, v->value);
2708
2709   return s;
2710 }
2711
2712 static u32
2713 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2714 {
2715   snat_main_t *sm = &snat_main;
2716   u32 next_worker_index = 0;
2717   u32 hash;
2718
2719   next_worker_index = sm->first_worker_index;
2720   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2721     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2722
2723   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2724     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2725   else
2726     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2727
2728   return next_worker_index;
2729 }
2730
2731 static u32
2732 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2733 {
2734   snat_main_t *sm = &snat_main;
2735   udp_header_t *udp;
2736   u16 port;
2737   snat_session_key_t m_key;
2738   clib_bihash_kv_8_8_t kv, value;
2739   snat_static_mapping_t *m;
2740   u32 proto;
2741   u32 next_worker_index = 0;
2742
2743   /* first try static mappings without port */
2744   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2745     {
2746       m_key.addr = ip0->dst_address;
2747       m_key.port = 0;
2748       m_key.protocol = 0;
2749       m_key.fib_index = rx_fib_index0;
2750       kv.key = m_key.as_u64;
2751       if (!clib_bihash_search_8_8
2752           (&sm->static_mapping_by_external, &kv, &value))
2753         {
2754           m = pool_elt_at_index (sm->static_mappings, value.value);
2755           return m->workers[0];
2756         }
2757     }
2758
2759   proto = ip_proto_to_snat_proto (ip0->protocol);
2760   udp = ip4_next_header (ip0);
2761   port = udp->dst_port;
2762
2763   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2764     {
2765       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2766         return vlib_get_thread_index ();
2767
2768       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2769         {
2770           nat_reass_ip4_t *reass;
2771
2772           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2773                                       ip0->fragment_id, ip0->protocol);
2774
2775           if (reass && (reass->thread_index != (u32) ~ 0))
2776             return reass->thread_index;
2777           else
2778             return vlib_get_thread_index ();
2779         }
2780     }
2781
2782   /* unknown protocol */
2783   if (PREDICT_FALSE (proto == ~0))
2784     {
2785       /* use current thread */
2786       return vlib_get_thread_index ();
2787     }
2788
2789   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2790     {
2791       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2792       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2793       if (!icmp_is_error_message (icmp))
2794         port = echo->identifier;
2795       else
2796         {
2797           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2798           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2799           void *l4_header = ip4_next_header (inner_ip);
2800           switch (proto)
2801             {
2802             case SNAT_PROTOCOL_ICMP:
2803               icmp = (icmp46_header_t *) l4_header;
2804               echo = (icmp_echo_header_t *) (icmp + 1);
2805               port = echo->identifier;
2806               break;
2807             case SNAT_PROTOCOL_UDP:
2808             case SNAT_PROTOCOL_TCP:
2809               port = ((tcp_udp_header_t *) l4_header)->src_port;
2810               break;
2811             default:
2812               return vlib_get_thread_index ();
2813             }
2814         }
2815     }
2816
2817   /* try static mappings with port */
2818   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2819     {
2820       m_key.addr = ip0->dst_address;
2821       m_key.port = clib_net_to_host_u16 (port);
2822       m_key.protocol = proto;
2823       m_key.fib_index = rx_fib_index0;
2824       kv.key = m_key.as_u64;
2825       if (!clib_bihash_search_8_8
2826           (&sm->static_mapping_by_external, &kv, &value))
2827         {
2828           m = pool_elt_at_index (sm->static_mappings, value.value);
2829           return m->workers[0];
2830         }
2831     }
2832
2833   /* worker by outside port */
2834   next_worker_index = sm->first_worker_index;
2835   next_worker_index +=
2836     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2837   return next_worker_index;
2838 }
2839
2840 static u32
2841 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2842 {
2843   snat_main_t *sm = &snat_main;
2844   clib_bihash_kv_8_8_t kv, value;
2845   u32 proto, next_worker_index = 0;
2846   udp_header_t *udp;
2847   u16 port;
2848   snat_static_mapping_t *m;
2849   u32 hash;
2850
2851   /* first try static mappings without port */
2852   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2853     {
2854       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2855       if (!clib_bihash_search_8_8
2856           (&sm->static_mapping_by_external, &kv, &value))
2857         {
2858           m = pool_elt_at_index (sm->static_mappings, value.value);
2859           return m->workers[0];
2860         }
2861     }
2862
2863   proto = ip_proto_to_snat_proto (ip->protocol);
2864
2865   /* unknown protocol */
2866   if (PREDICT_FALSE (proto == ~0))
2867     {
2868       /* use current thread */
2869       return vlib_get_thread_index ();
2870     }
2871
2872   udp = ip4_next_header (ip);
2873   port = udp->dst_port;
2874
2875   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2876     {
2877       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2878       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2879       if (!icmp_is_error_message (icmp))
2880         port = echo->identifier;
2881       else
2882         {
2883           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2884           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2885           void *l4_header = ip4_next_header (inner_ip);
2886           switch (proto)
2887             {
2888             case SNAT_PROTOCOL_ICMP:
2889               icmp = (icmp46_header_t *) l4_header;
2890               echo = (icmp_echo_header_t *) (icmp + 1);
2891               port = echo->identifier;
2892               break;
2893             case SNAT_PROTOCOL_UDP:
2894             case SNAT_PROTOCOL_TCP:
2895               port = ((tcp_udp_header_t *) l4_header)->src_port;
2896               break;
2897             default:
2898               return vlib_get_thread_index ();
2899             }
2900         }
2901     }
2902
2903   /* try static mappings with port */
2904   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2905     {
2906       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2907                   clib_net_to_host_u16 (port));
2908       if (!clib_bihash_search_8_8
2909           (&sm->static_mapping_by_external, &kv, &value))
2910         {
2911           m = pool_elt_at_index (sm->static_mappings, value.value);
2912           if (!is_lb_static_mapping (m))
2913             return m->workers[0];
2914
2915           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2916             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
2917
2918           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2919             return m->workers[hash & (_vec_len (m->workers) - 1)];
2920           else
2921             return m->workers[hash % _vec_len (m->workers)];
2922         }
2923     }
2924
2925   /* worker by outside port */
2926   next_worker_index = sm->first_worker_index;
2927   next_worker_index +=
2928     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2929
2930   return next_worker_index;
2931 }
2932
2933 static clib_error_t *
2934 snat_config (vlib_main_t * vm, unformat_input_t * input)
2935 {
2936   snat_main_t *sm = &snat_main;
2937   nat66_main_t *nm = &nat66_main;
2938   u32 translation_buckets = 1024;
2939   u32 translation_memory_size = 128 << 20;
2940   u32 user_buckets = 128;
2941   u32 user_memory_size = 64 << 20;
2942   u32 max_translations_per_user = 100;
2943   u32 outside_vrf_id = 0;
2944   u32 outside_ip6_vrf_id = 0;
2945   u32 inside_vrf_id = 0;
2946   u32 static_mapping_buckets = 1024;
2947   u32 static_mapping_memory_size = 64 << 20;
2948   u32 nat64_bib_buckets = 1024;
2949   u32 nat64_bib_memory_size = 128 << 20;
2950   u32 nat64_st_buckets = 2048;
2951   u32 nat64_st_memory_size = 256 << 20;
2952   u8 static_mapping_only = 0;
2953   u8 static_mapping_connection_tracking = 0;
2954   snat_main_per_thread_data_t *tsm;
2955   dslite_main_t *dm = &dslite_main;
2956
2957   sm->deterministic = 0;
2958   sm->out2in_dpo = 0;
2959   sm->endpoint_dependent = 0;
2960
2961   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2962     {
2963       if (unformat
2964           (input, "translation hash buckets %d", &translation_buckets))
2965         ;
2966       else if (unformat (input, "translation hash memory %d",
2967                          &translation_memory_size));
2968       else if (unformat (input, "user hash buckets %d", &user_buckets))
2969         ;
2970       else if (unformat (input, "user hash memory %d", &user_memory_size))
2971         ;
2972       else if (unformat (input, "max translations per user %d",
2973                          &max_translations_per_user))
2974         ;
2975       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
2976         ;
2977       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
2978         ;
2979       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
2980         ;
2981       else if (unformat (input, "static mapping only"))
2982         {
2983           static_mapping_only = 1;
2984           if (unformat (input, "connection tracking"))
2985             static_mapping_connection_tracking = 1;
2986         }
2987       else if (unformat (input, "deterministic"))
2988         sm->deterministic = 1;
2989       else if (unformat (input, "nat64 bib hash buckets %d",
2990                          &nat64_bib_buckets))
2991         ;
2992       else if (unformat (input, "nat64 bib hash memory %d",
2993                          &nat64_bib_memory_size))
2994         ;
2995       else
2996         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2997         ;
2998       else if (unformat (input, "nat64 st hash memory %d",
2999                          &nat64_st_memory_size))
3000         ;
3001       else if (unformat (input, "out2in dpo"))
3002         sm->out2in_dpo = 1;
3003       else if (unformat (input, "dslite ce"))
3004         dslite_set_ce (dm, 1);
3005       else if (unformat (input, "endpoint-dependent"))
3006         sm->endpoint_dependent = 1;
3007       else
3008         return clib_error_return (0, "unknown input '%U'",
3009                                   format_unformat_error, input);
3010     }
3011
3012   if (sm->deterministic && sm->endpoint_dependent)
3013     return clib_error_return (0,
3014                               "deterministic and endpoint-dependent modes are mutually exclusive");
3015
3016   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3017     return clib_error_return (0,
3018                               "static mapping only mode available only for simple nat");
3019
3020   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3021     return clib_error_return (0,
3022                               "out2in dpo mode available only for simple nat");
3023
3024   /* for show commands, etc. */
3025   sm->translation_buckets = translation_buckets;
3026   sm->translation_memory_size = translation_memory_size;
3027   /* do not exceed load factor 10 */
3028   sm->max_translations = 10 * translation_buckets;
3029   sm->user_buckets = user_buckets;
3030   sm->user_memory_size = user_memory_size;
3031   sm->max_translations_per_user = max_translations_per_user;
3032   sm->outside_vrf_id = outside_vrf_id;
3033   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3034                                                              outside_vrf_id,
3035                                                              FIB_SOURCE_PLUGIN_HI);
3036   nm->outside_vrf_id = outside_ip6_vrf_id;
3037   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3038                                                              outside_ip6_vrf_id,
3039                                                              FIB_SOURCE_PLUGIN_HI);
3040   sm->inside_vrf_id = inside_vrf_id;
3041   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3042                                                             inside_vrf_id,
3043                                                             FIB_SOURCE_PLUGIN_HI);
3044   sm->static_mapping_only = static_mapping_only;
3045   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3046
3047   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3048                   nat64_st_memory_size);
3049
3050   if (sm->deterministic)
3051     {
3052       sm->in2out_node_index = snat_det_in2out_node.index;
3053       sm->in2out_output_node_index = ~0;
3054       sm->out2in_node_index = snat_det_out2in_node.index;
3055       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3056       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3057     }
3058   else
3059     {
3060       if (sm->endpoint_dependent)
3061         {
3062           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3063           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3064           sm->in2out_node_index = nat44_ed_in2out_node.index;
3065           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3066           sm->out2in_node_index = nat44_ed_out2in_node.index;
3067           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3068           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3069           nat_affinity_init (vm);
3070         }
3071       else
3072         {
3073           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3074           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3075           sm->in2out_node_index = snat_in2out_node.index;
3076           sm->in2out_output_node_index = snat_in2out_output_node.index;
3077           sm->out2in_node_index = snat_out2in_node.index;
3078           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3079           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3080         }
3081       if (!static_mapping_only ||
3082           (static_mapping_only && static_mapping_connection_tracking))
3083         {
3084           /* *INDENT-OFF* */
3085           vec_foreach (tsm, sm->per_thread_data)
3086             {
3087               if (sm->endpoint_dependent)
3088                 {
3089                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3090                                          translation_buckets,
3091                                          translation_memory_size);
3092                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3093                                                       format_ed_session_kvp);
3094
3095                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3096                                          translation_buckets,
3097                                          translation_memory_size);
3098                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3099                                                       format_ed_session_kvp);
3100                 }
3101               else
3102                 {
3103                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3104                                         translation_buckets,
3105                                         translation_memory_size);
3106                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3107                                                      format_session_kvp);
3108
3109                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3110                                         translation_buckets,
3111                                         translation_memory_size);
3112                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3113                                                      format_session_kvp);
3114                 }
3115
3116               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3117                                     user_memory_size);
3118               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3119                                                  format_user_kvp);
3120             }
3121           /* *INDENT-ON* */
3122
3123         }
3124       else
3125         {
3126           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3127           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3128         }
3129       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3130                             "static_mapping_by_local", static_mapping_buckets,
3131                             static_mapping_memory_size);
3132       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3133                                          format_static_mapping_kvp);
3134
3135       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3136                             "static_mapping_by_external",
3137                             static_mapping_buckets,
3138                             static_mapping_memory_size);
3139       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3140                                          format_static_mapping_kvp);
3141     }
3142
3143   return 0;
3144 }
3145
3146 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3147
3148 static void
3149 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3150                                  uword opaque,
3151                                  u32 sw_if_index,
3152                                  ip4_address_t * address,
3153                                  u32 address_length,
3154                                  u32 if_address_index, u32 is_delete)
3155 {
3156   snat_main_t *sm = &snat_main;
3157   snat_static_map_resolve_t *rp;
3158   snat_static_mapping_t *m;
3159   snat_session_key_t m_key;
3160   clib_bihash_kv_8_8_t kv, value;
3161   int i, rv;
3162   ip4_address_t l_addr;
3163
3164   for (i = 0; i < vec_len (sm->to_resolve); i++)
3165     {
3166       rp = sm->to_resolve + i;
3167       if (rp->addr_only == 0)
3168         continue;
3169       if (rp->sw_if_index == sw_if_index)
3170         goto match;
3171     }
3172
3173   return;
3174
3175 match:
3176   m_key.addr.as_u32 = address->as_u32;
3177   m_key.port = rp->addr_only ? 0 : rp->e_port;
3178   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3179   m_key.fib_index = sm->outside_fib_index;
3180   kv.key = m_key.as_u64;
3181   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3182     m = 0;
3183   else
3184     m = pool_elt_at_index (sm->static_mappings, value.value);
3185
3186   if (!is_delete)
3187     {
3188       /* Don't trip over lease renewal, static config */
3189       if (m)
3190         return;
3191     }
3192   else
3193     {
3194       if (!m)
3195         return;
3196     }
3197
3198   /* Indetity mapping? */
3199   if (rp->l_addr.as_u32 == 0)
3200     l_addr.as_u32 = address[0].as_u32;
3201   else
3202     l_addr.as_u32 = rp->l_addr.as_u32;
3203   /* Add the static mapping */
3204   rv = snat_add_static_mapping (l_addr,
3205                                 address[0],
3206                                 rp->l_port,
3207                                 rp->e_port,
3208                                 rp->vrf_id,
3209                                 rp->addr_only, ~0 /* sw_if_index */ ,
3210                                 rp->proto, !is_delete, rp->twice_nat,
3211                                 rp->out2in_only, rp->tag, rp->identity_nat);
3212   if (rv)
3213     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3214 }
3215
3216 static void
3217 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3218                                        uword opaque,
3219                                        u32 sw_if_index,
3220                                        ip4_address_t * address,
3221                                        u32 address_length,
3222                                        u32 if_address_index, u32 is_delete)
3223 {
3224   snat_main_t *sm = &snat_main;
3225   snat_static_map_resolve_t *rp;
3226   ip4_address_t l_addr;
3227   int i, j;
3228   int rv;
3229   u8 twice_nat = 0;
3230   snat_address_t *addresses = sm->addresses;
3231
3232   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3233     {
3234       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3235         goto match;
3236     }
3237
3238   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3239     {
3240       twice_nat = 1;
3241       addresses = sm->twice_nat_addresses;
3242       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3243         goto match;
3244     }
3245
3246   return;
3247
3248 match:
3249   if (!is_delete)
3250     {
3251       /* Don't trip over lease renewal, static config */
3252       for (j = 0; j < vec_len (addresses); j++)
3253         if (addresses[j].addr.as_u32 == address->as_u32)
3254           return;
3255
3256       (void) snat_add_address (sm, address, ~0, twice_nat);
3257       /* Scan static map resolution vector */
3258       for (j = 0; j < vec_len (sm->to_resolve); j++)
3259         {
3260           rp = sm->to_resolve + j;
3261           if (rp->addr_only)
3262             continue;
3263           /* On this interface? */
3264           if (rp->sw_if_index == sw_if_index)
3265             {
3266               /* Indetity mapping? */
3267               if (rp->l_addr.as_u32 == 0)
3268                 l_addr.as_u32 = address[0].as_u32;
3269               else
3270                 l_addr.as_u32 = rp->l_addr.as_u32;
3271               /* Add the static mapping */
3272               rv = snat_add_static_mapping (l_addr,
3273                                             address[0],
3274                                             rp->l_port,
3275                                             rp->e_port,
3276                                             rp->vrf_id,
3277                                             rp->addr_only,
3278                                             ~0 /* sw_if_index */ ,
3279                                             rp->proto,
3280                                             rp->is_add, rp->twice_nat,
3281                                             rp->out2in_only, rp->tag,
3282                                             rp->identity_nat);
3283               if (rv)
3284                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3285             }
3286         }
3287       return;
3288     }
3289   else
3290     {
3291       (void) snat_del_address (sm, address[0], 1, twice_nat);
3292       return;
3293     }
3294 }
3295
3296
3297 int
3298 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3299                             u8 twice_nat)
3300 {
3301   ip4_main_t *ip4_main = sm->ip4_main;
3302   ip4_address_t *first_int_addr;
3303   snat_static_map_resolve_t *rp;
3304   u32 *indices_to_delete = 0;
3305   int i, j;
3306   u32 *auto_add_sw_if_indices =
3307     twice_nat ? sm->
3308     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3309
3310   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3311     );
3312
3313   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3314     {
3315       if (auto_add_sw_if_indices[i] == sw_if_index)
3316         {
3317           if (is_del)
3318             {
3319               /* if have address remove it */
3320               if (first_int_addr)
3321                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3322               else
3323                 {
3324                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3325                     {
3326                       rp = sm->to_resolve + j;
3327                       if (rp->sw_if_index == sw_if_index)
3328                         vec_add1 (indices_to_delete, j);
3329                     }
3330                   if (vec_len (indices_to_delete))
3331                     {
3332                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3333                         vec_del1 (sm->to_resolve, j);
3334                       vec_free (indices_to_delete);
3335                     }
3336                 }
3337               if (twice_nat)
3338                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3339               else
3340                 vec_del1 (sm->auto_add_sw_if_indices, i);
3341             }
3342           else
3343             return VNET_API_ERROR_VALUE_EXIST;
3344
3345           return 0;
3346         }
3347     }
3348
3349   if (is_del)
3350     return VNET_API_ERROR_NO_SUCH_ENTRY;
3351
3352   /* add to the auto-address list */
3353   if (twice_nat)
3354     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3355   else
3356     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3357
3358   /* If the address is already bound - or static - add it now */
3359   if (first_int_addr)
3360     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3361
3362   return 0;
3363 }
3364
3365 int
3366 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3367                    snat_protocol_t proto, u32 vrf_id, int is_in)
3368 {
3369   snat_main_per_thread_data_t *tsm;
3370   clib_bihash_kv_8_8_t kv, value;
3371   ip4_header_t ip;
3372   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3373   snat_session_key_t key;
3374   snat_session_t *s;
3375   clib_bihash_8_8_t *t;
3376
3377   if (sm->endpoint_dependent)
3378     return VNET_API_ERROR_UNSUPPORTED;
3379
3380   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3381   if (sm->num_workers > 1)
3382     tsm =
3383       vec_elt_at_index (sm->per_thread_data,
3384                         sm->worker_in2out_cb (&ip, fib_index));
3385   else
3386     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3387
3388   key.addr.as_u32 = addr->as_u32;
3389   key.port = clib_host_to_net_u16 (port);
3390   key.protocol = proto;
3391   key.fib_index = fib_index;
3392   kv.key = key.as_u64;
3393   t = is_in ? &tsm->in2out : &tsm->out2in;
3394   if (!clib_bihash_search_8_8 (t, &kv, &value))
3395     {
3396       if (pool_is_free_index (tsm->sessions, value.value))
3397         return VNET_API_ERROR_UNSPECIFIED;
3398
3399       s = pool_elt_at_index (tsm->sessions, value.value);
3400       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3401       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3402       return 0;
3403     }
3404
3405   return VNET_API_ERROR_NO_SUCH_ENTRY;
3406 }
3407
3408 int
3409 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3410                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3411                       u32 vrf_id, int is_in)
3412 {
3413   ip4_header_t ip;
3414   clib_bihash_16_8_t *t;
3415   nat_ed_ses_key_t key;
3416   clib_bihash_kv_16_8_t kv, value;
3417   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3418   snat_session_t *s;
3419   snat_main_per_thread_data_t *tsm;
3420
3421   if (!sm->endpoint_dependent)
3422     return VNET_API_ERROR_FEATURE_DISABLED;
3423
3424   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3425   if (sm->num_workers > 1)
3426     tsm =
3427       vec_elt_at_index (sm->per_thread_data,
3428                         sm->worker_in2out_cb (&ip, fib_index));
3429   else
3430     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3431
3432   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3433   key.l_addr.as_u32 = addr->as_u32;
3434   key.r_addr.as_u32 = eh_addr->as_u32;
3435   key.l_port = clib_host_to_net_u16 (port);
3436   key.r_port = clib_host_to_net_u16 (eh_port);
3437   key.proto = proto;
3438   key.fib_index = fib_index;
3439   kv.key[0] = key.as_u64[0];
3440   kv.key[1] = key.as_u64[1];
3441   if (clib_bihash_search_16_8 (t, &kv, &value))
3442     return VNET_API_ERROR_NO_SUCH_ENTRY;
3443
3444   if (pool_is_free_index (tsm->sessions, value.value))
3445     return VNET_API_ERROR_UNSPECIFIED;
3446   s = pool_elt_at_index (tsm->sessions, value.value);
3447   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3448   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3449   return 0;
3450 }
3451
3452 void
3453 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3454 {
3455   snat_main_t *sm = &snat_main;
3456
3457   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3458   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3459   sm->psid = psid;
3460   sm->psid_offset = psid_offset;
3461   sm->psid_length = psid_length;
3462 }
3463
3464 void
3465 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3466 {
3467   snat_main_t *sm = &snat_main;
3468
3469   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3470   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3471   sm->start_port = start_port;
3472   sm->end_port = end_port;
3473 }
3474
3475 void
3476 nat_set_alloc_addr_and_port_default (void)
3477 {
3478   snat_main_t *sm = &snat_main;
3479
3480   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3481   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3482 }
3483
3484 /*
3485  * fd.io coding-style-patch-verification: ON
3486  *
3487  * Local Variables:
3488  * eval: (c-set-style "gnu")
3489  * End:
3490  */