src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <vnet/fib/fib_table.h>
  34 #include <vnet/fib/ip4_fib.h>
  35
  36 #include <vpp/app/version.h>
  37
  38 snat_main_t snat_main;
  39
  40 /* *INDENT-OFF* */
  41
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat44-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  47 };
  48 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat44-out2in",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-dhcp-client-detect"),
  53 };
  54 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-classify",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-in2out",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-out2in",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  68                                "ip4-dhcp-client-detect"),
  69 };
  70 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-det-classify",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  74 };
  75 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  76   .arc_name = "ip4-unicast",
  77   .node_name = "nat44-ed-in2out",
  78   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-ed-out2in",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  84                                "ip4-dhcp-client-detect"),
  85 };
  86 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-ed-classify",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  90 };
  91 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-in2out-worker-handoff",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  95 };
  96 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-out2in-worker-handoff",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-handoff-classify",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-in2out-fast",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 111 };
 112 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-out2in-fast",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 116                                "ip4-dhcp-client-detect"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-hairpin-dst",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ed-hairpin-dst",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 127 };
 128
 129 /* Hook up output features */
 130 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 131   .arc_name = "ip4-output",
 132   .node_name = "nat44-in2out-output",
 133   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 134 };
 135 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 136   .arc_name = "ip4-output",
 137   .node_name = "nat44-in2out-output-worker-handoff",
 138   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 139 };
 140 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 141   .arc_name = "ip4-output",
 142   .node_name = "nat44-hairpin-src",
 143   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 144 };
 145 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-ed-in2out-output",
 148   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 149 };
 150 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-ed-hairpin-src",
 153   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 154 };
 155
 156 /* Hook up ip4-local features */
 157 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 158 {
 159   .arc_name = "ip4-local",
 160   .node_name = "nat44-hairpinning",
 161   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 162 };
 163 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 164 {
 165   .arc_name = "ip4-local",
 166   .node_name = "nat44-ed-hairpinning",
 167   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 168 };
 169
 170
 171 VLIB_PLUGIN_REGISTER () = {
 172     .version = VPP_BUILD_VER,
 173     .description = "Network Address Translation",
 174 };
 175 /* *INDENT-ON* */
 176
 177 void
 178 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 179 {
 180   snat_session_key_t key;
 181   clib_bihash_kv_8_8_t kv;
 182   nat_ed_ses_key_t ed_key;
 183   clib_bihash_kv_16_8_t ed_kv;
 184   snat_main_per_thread_data_t *tsm =
 185     vec_elt_at_index (sm->per_thread_data, thread_index);
 186
 187   if (is_fwd_bypass_session (s))
 188     {
 189       ed_key.l_addr = s->in2out.addr;
 190       ed_key.r_addr = s->ext_host_addr;
 191       ed_key.l_port = s->in2out.port;
 192       ed_key.r_port = s->ext_host_port;
 193       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 194       ed_key.fib_index = 0;
 195       ed_kv.key[0] = ed_key.as_u64[0];
 196       ed_kv.key[1] = ed_key.as_u64[1];
 197       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 198         nat_log_warn ("in2out_ed key del failed");
 199       return;
 200     }
 201
 202   /* session lookup tables */
 203   if (is_ed_session (s))
 204     {
 205       if (is_affinity_sessions (s))
 206         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 207                              s->in2out.protocol, s->out2in.port);
 208       ed_key.l_addr = s->out2in.addr;
 209       ed_key.r_addr = s->ext_host_addr;
 210       ed_key.fib_index = s->out2in.fib_index;
 211       if (snat_is_unk_proto_session (s))
 212         {
 213           ed_key.proto = s->in2out.port;
 214           ed_key.r_port = 0;
 215           ed_key.l_port = 0;
 216         }
 217       else
 218         {
 219           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 220           ed_key.l_port = s->out2in.port;
 221           ed_key.r_port = s->ext_host_port;
 222         }
 223       ed_kv.key[0] = ed_key.as_u64[0];
 224       ed_kv.key[1] = ed_key.as_u64[1];
 225       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 226         nat_log_warn ("out2in_ed key del failed");
 227       ed_key.l_addr = s->in2out.addr;
 228       ed_key.fib_index = s->in2out.fib_index;
 229       if (!snat_is_unk_proto_session (s))
 230         ed_key.l_port = s->in2out.port;
 231       if (is_twice_nat_session (s))
 232         {
 233           ed_key.r_addr = s->ext_host_nat_addr;
 234           ed_key.r_port = s->ext_host_nat_port;
 235         }
 236       ed_kv.key[0] = ed_key.as_u64[0];
 237       ed_kv.key[1] = ed_key.as_u64[1];
 238       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 239         nat_log_warn ("in2out_ed key del failed");
 240
 241       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 242                              &s->in2out.addr, s->in2out.port,
 243                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 244                              &s->out2in.addr, s->out2in.port,
 245                              &s->ext_host_addr, s->ext_host_port,
 246                              s->in2out.protocol, is_twice_nat_session (s));
 247     }
 248   else
 249     {
 250       kv.key = s->in2out.as_u64;
 251       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 252         nat_log_warn ("in2out key del failed");
 253       kv.key = s->out2in.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 255         nat_log_warn ("out2in key del failed");
 256
 257       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 258                                &s->in2out.addr, s->in2out.port,
 259                                &s->out2in.addr, s->out2in.port,
 260                                s->in2out.protocol);
 261     }
 262
 263   if (snat_is_unk_proto_session (s))
 264     return;
 265
 266   /* log NAT event */
 267   snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 268                                        s->out2in.addr.as_u32,
 269                                        s->in2out.protocol,
 270                                        s->in2out.port,
 271                                        s->out2in.port, s->in2out.fib_index);
 272
 273   /* Twice NAT address and port for external host */
 274   if (is_twice_nat_session (s))
 275     {
 276       key.protocol = s->in2out.protocol;
 277       key.port = s->ext_host_nat_port;
 278       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 279       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 280                                           thread_index, &key);
 281     }
 282
 283   if (snat_is_session_static (s))
 284     return;
 285
 286   snat_free_outside_address_and_port (sm->addresses, thread_index,
 287                                       &s->out2in);
 288 }
 289
 290 snat_user_t *
 291 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 292                         u32 thread_index)
 293 {
 294   snat_user_t *u = 0;
 295   snat_user_key_t user_key;
 296   clib_bihash_kv_8_8_t kv, value;
 297   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 298   dlist_elt_t *per_user_list_head_elt;
 299
 300   user_key.addr.as_u32 = addr->as_u32;
 301   user_key.fib_index = fib_index;
 302   kv.key = user_key.as_u64;
 303
 304   /* Ever heard of the "user" = src ip4 address before? */
 305   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 306     {
 307       /* no, make a new one */
 308       pool_get (tsm->users, u);
 309       clib_memset (u, 0, sizeof (*u));
 310       u->addr.as_u32 = addr->as_u32;
 311       u->fib_index = fib_index;
 312
 313       pool_get (tsm->list_pool, per_user_list_head_elt);
 314
 315       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 316         tsm->list_pool;
 317
 318       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 319
 320       kv.value = u - tsm->users;
 321
 322       /* add user */
 323       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 324         nat_log_warn ("user_hash keay add failed");
 325
 326       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 327                                pool_elts (tsm->users));
 328     }
 329   else
 330     {
 331       u = pool_elt_at_index (tsm->users, value.value);
 332     }
 333
 334   return u;
 335 }
 336
 337 snat_session_t *
 338 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 339                               u32 thread_index)
 340 {
 341   snat_session_t *s;
 342   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 343   u32 oldest_per_user_translation_list_index, session_index;
 344   dlist_elt_t *oldest_per_user_translation_list_elt;
 345   dlist_elt_t *per_user_translation_list_elt;
 346
 347   /* Over quota? Recycle the least recently used translation */
 348   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 349     {
 350       oldest_per_user_translation_list_index =
 351         clib_dlist_remove_head (tsm->list_pool,
 352                                 u->sessions_per_user_list_head_index);
 353
 354       ASSERT (oldest_per_user_translation_list_index != ~0);
 355
 356       /* Add it back to the end of the LRU list */
 357       clib_dlist_addtail (tsm->list_pool,
 358                           u->sessions_per_user_list_head_index,
 359                           oldest_per_user_translation_list_index);
 360       /* Get the list element */
 361       oldest_per_user_translation_list_elt =
 362         pool_elt_at_index (tsm->list_pool,
 363                            oldest_per_user_translation_list_index);
 364
 365       /* Get the session index from the list element */
 366       session_index = oldest_per_user_translation_list_elt->value;
 367
 368       /* Get the session */
 369       s = pool_elt_at_index (tsm->sessions, session_index);
 370       nat_free_session_data (sm, s, thread_index);
 371       if (snat_is_session_static (s))
 372         u->nstaticsessions--;
 373       else
 374         u->nsessions--;
 375       s->flags = 0;
 376       s->total_bytes = 0;
 377       s->total_pkts = 0;
 378       s->state = 0;
 379       s->ext_host_addr.as_u32 = 0;
 380       s->ext_host_port = 0;
 381       s->ext_host_nat_addr.as_u32 = 0;
 382       s->ext_host_nat_port = 0;
 383     }
 384   else
 385     {
 386       pool_get (tsm->sessions, s);
 387       clib_memset (s, 0, sizeof (*s));
 388
 389       /* Create list elts */
 390       pool_get (tsm->list_pool, per_user_translation_list_elt);
 391       clib_dlist_init (tsm->list_pool,
 392                        per_user_translation_list_elt - tsm->list_pool);
 393
 394       per_user_translation_list_elt->value = s - tsm->sessions;
 395       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 396       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 397
 398       clib_dlist_addtail (tsm->list_pool,
 399                           s->per_user_list_head_index,
 400                           per_user_translation_list_elt - tsm->list_pool);
 401
 402       s->user_index = u - tsm->users;
 403       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 404                                pool_elts (tsm->sessions));
 405     }
 406
 407   return s;
 408 }
 409
 410 snat_session_t *
 411 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 412                       f64 now)
 413 {
 414   snat_session_t *s;
 415   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 416   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 417   u32 oldest_index;
 418   u64 sess_timeout_time;
 419
 420   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 421     goto alloc_new;
 422
 423   oldest_index =
 424     clib_dlist_remove_head (tsm->list_pool,
 425                             u->sessions_per_user_list_head_index);
 426   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 427   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 428   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 429   if (now >= sess_timeout_time)
 430     {
 431       clib_dlist_addtail (tsm->list_pool,
 432                           u->sessions_per_user_list_head_index, oldest_index);
 433       nat_free_session_data (sm, s, thread_index);
 434       if (snat_is_session_static (s))
 435         u->nstaticsessions--;
 436       else
 437         u->nsessions--;
 438       s->flags = 0;
 439       s->total_bytes = 0;
 440       s->total_pkts = 0;
 441       s->state = 0;
 442       s->ext_host_addr.as_u32 = 0;
 443       s->ext_host_port = 0;
 444       s->ext_host_nat_addr.as_u32 = 0;
 445       s->ext_host_nat_port = 0;
 446     }
 447   else
 448     {
 449       clib_dlist_addhead (tsm->list_pool,
 450                           u->sessions_per_user_list_head_index, oldest_index);
 451       if ((u->nsessions + u->nstaticsessions) >=
 452           sm->max_translations_per_user)
 453         {
 454           nat_log_warn ("max translations per user %U", format_ip4_address,
 455                         &u->addr);
 456           snat_ipfix_logging_max_entries_per_user
 457             (sm->max_translations_per_user, u->addr.as_u32);
 458           return 0;
 459         }
 460       else
 461         {
 462         alloc_new:
 463           pool_get (tsm->sessions, s);
 464           clib_memset (s, 0, sizeof (*s));
 465
 466           /* Create list elts */
 467           pool_get (tsm->list_pool, per_user_translation_list_elt);
 468           clib_dlist_init (tsm->list_pool,
 469                            per_user_translation_list_elt - tsm->list_pool);
 470
 471           per_user_translation_list_elt->value = s - tsm->sessions;
 472           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 473           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 474
 475           clib_dlist_addtail (tsm->list_pool,
 476                               s->per_user_list_head_index,
 477                               per_user_translation_list_elt - tsm->list_pool);
 478         }
 479
 480       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 481                                pool_elts (tsm->sessions));
 482     }
 483
 484   return s;
 485 }
 486
 487 void
 488 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 489                           int is_add)
 490 {
 491   fib_prefix_t prefix = {
 492     .fp_len = p_len,
 493     .fp_proto = FIB_PROTOCOL_IP4,
 494     .fp_addr = {
 495                 .ip4.as_u32 = addr->as_u32,
 496                 },
 497   };
 498   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 499
 500   if (is_add)
 501     fib_table_entry_update_one_path (fib_index,
 502                                      &prefix,
 503                                      FIB_SOURCE_PLUGIN_LOW,
 504                                      (FIB_ENTRY_FLAG_CONNECTED |
 505                                       FIB_ENTRY_FLAG_LOCAL |
 506                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 507                                      DPO_PROTO_IP4,
 508                                      NULL,
 509                                      sw_if_index,
 510                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 511   else
 512     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 513 }
 514
 515 int
 516 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 517                   u8 twice_nat)
 518 {
 519   snat_address_t *ap;
 520   snat_interface_t *i;
 521   vlib_thread_main_t *tm = vlib_get_thread_main ();
 522
 523   if (twice_nat && !sm->endpoint_dependent)
 524     return VNET_API_ERROR_FEATURE_DISABLED;
 525
 526   /* Check if address already exists */
 527   /* *INDENT-OFF* */
 528   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 529     {
 530       if (ap->addr.as_u32 == addr->as_u32)
 531         return VNET_API_ERROR_VALUE_EXIST;
 532     }
 533   /* *INDENT-ON* */
 534
 535   if (twice_nat)
 536     vec_add2 (sm->twice_nat_addresses, ap, 1);
 537   else
 538     vec_add2 (sm->addresses, ap, 1);
 539
 540   ap->addr = *addr;
 541   if (vrf_id != ~0)
 542     ap->fib_index =
 543       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 544                                          FIB_SOURCE_PLUGIN_LOW);
 545   else
 546     ap->fib_index = ~0;
 547 #define _(N, i, n, s) \
 548   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 549   ap->busy_##n##_ports = 0; \
 550   ap->busy_##n##_ports_per_thread = 0;\
 551   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 552   foreach_snat_protocol
 553 #undef _
 554     if (twice_nat)
 555     return 0;
 556
 557   /* Add external address to FIB */
 558   /* *INDENT-OFF* */
 559   pool_foreach (i, sm->interfaces,
 560   ({
 561     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 562       continue;
 563
 564     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 565     break;
 566   }));
 567   pool_foreach (i, sm->output_feature_interfaces,
 568   ({
 569     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 570       continue;
 571
 572     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 573     break;
 574   }));
 575   /* *INDENT-ON* */
 576
 577   return 0;
 578 }
 579
 580 static int
 581 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 582 {
 583   snat_static_mapping_t *m;
 584   /* *INDENT-OFF* */
 585   pool_foreach (m, sm->static_mappings,
 586   ({
 587       if (is_addr_only_static_mapping (m) ||
 588           is_out2in_only_static_mapping (m) ||
 589           is_identity_static_mapping (m))
 590         continue;
 591       if (m->external_addr.as_u32 == addr.as_u32)
 592         return 1;
 593   }));
 594   /* *INDENT-ON* */
 595
 596   return 0;
 597 }
 598
 599 void
 600 increment_v4_address (ip4_address_t * a)
 601 {
 602   u32 v;
 603
 604   v = clib_net_to_host_u32 (a->as_u32) + 1;
 605   a->as_u32 = clib_host_to_net_u32 (v);
 606 }
 607
 608 static void
 609 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 610                                        ip4_address_t l_addr,
 611                                        u16 l_port,
 612                                        u32 sw_if_index,
 613                                        u16 e_port,
 614                                        u32 vrf_id,
 615                                        snat_protocol_t proto,
 616                                        int addr_only, int is_add, u8 * tag,
 617                                        int twice_nat, int out2in_only,
 618                                        int identity_nat)
 619 {
 620   snat_static_map_resolve_t *rp;
 621
 622   vec_add2 (sm->to_resolve, rp, 1);
 623   rp->l_addr.as_u32 = l_addr.as_u32;
 624   rp->l_port = l_port;
 625   rp->sw_if_index = sw_if_index;
 626   rp->e_port = e_port;
 627   rp->vrf_id = vrf_id;
 628   rp->proto = proto;
 629   rp->addr_only = addr_only;
 630   rp->is_add = is_add;
 631   rp->twice_nat = twice_nat;
 632   rp->out2in_only = out2in_only;
 633   rp->identity_nat = identity_nat;
 634   rp->tag = vec_dup (tag);
 635 }
 636
 637 static u32
 638 get_thread_idx_by_port (u16 e_port)
 639 {
 640   snat_main_t *sm = &snat_main;
 641   u32 thread_idx = sm->num_workers;
 642   if (sm->num_workers > 1)
 643     {
 644       thread_idx =
 645         sm->first_worker_index +
 646         sm->workers[(e_port - 1024) / sm->port_per_thread];
 647     }
 648   return thread_idx;
 649 }
 650
 651 int
 652 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 653                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 654                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 655                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 656                          u8 identity_nat)
 657 {
 658   snat_main_t *sm = &snat_main;
 659   snat_static_mapping_t *m;
 660   snat_session_key_t m_key;
 661   clib_bihash_kv_8_8_t kv, value;
 662   snat_address_t *a = 0;
 663   u32 fib_index = ~0;
 664   snat_interface_t *interface;
 665   int i;
 666   snat_main_per_thread_data_t *tsm;
 667   snat_user_key_t u_key;
 668   snat_user_t *u;
 669   dlist_elt_t *head, *elt;
 670   u32 elt_index, head_index;
 671   u32 ses_index;
 672   u64 user_index;
 673   snat_session_t *s;
 674   snat_static_map_resolve_t *rp, *rp_match = 0;
 675   nat44_lb_addr_port_t *local;
 676   u32 find = ~0;
 677
 678   if (!sm->endpoint_dependent)
 679     {
 680       if (twice_nat || out2in_only)
 681         return VNET_API_ERROR_FEATURE_DISABLED;
 682     }
 683
 684   /* If the external address is a specific interface address */
 685   if (sw_if_index != ~0)
 686     {
 687       ip4_address_t *first_int_addr;
 688
 689       for (i = 0; i < vec_len (sm->to_resolve); i++)
 690         {
 691           rp = sm->to_resolve + i;
 692           if (rp->sw_if_index != sw_if_index ||
 693               rp->l_addr.as_u32 != l_addr.as_u32 ||
 694               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 695             continue;
 696
 697           if (!addr_only)
 698             {
 699               if (rp->l_port != l_port || rp->e_port != e_port
 700                   || rp->proto != proto)
 701                 continue;
 702             }
 703
 704           rp_match = rp;
 705           break;
 706         }
 707
 708       /* Might be already set... */
 709       first_int_addr = ip4_interface_first_address
 710         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 711
 712       if (is_add)
 713         {
 714           if (rp_match)
 715             return VNET_API_ERROR_VALUE_EXIST;
 716
 717           snat_add_static_mapping_when_resolved
 718             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 719              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 720
 721           /* DHCP resolution required? */
 722           if (first_int_addr == 0)
 723             {
 724               return 0;
 725             }
 726           else
 727             {
 728               e_addr.as_u32 = first_int_addr->as_u32;
 729               /* Identity mapping? */
 730               if (l_addr.as_u32 == 0)
 731                 l_addr.as_u32 = e_addr.as_u32;
 732             }
 733         }
 734       else
 735         {
 736           if (!rp_match)
 737             return VNET_API_ERROR_NO_SUCH_ENTRY;
 738
 739           vec_del1 (sm->to_resolve, i);
 740
 741           if (first_int_addr)
 742             {
 743               e_addr.as_u32 = first_int_addr->as_u32;
 744               /* Identity mapping? */
 745               if (l_addr.as_u32 == 0)
 746                 l_addr.as_u32 = e_addr.as_u32;
 747             }
 748           else
 749             return 0;
 750         }
 751     }
 752
 753   m_key.addr = e_addr;
 754   m_key.port = addr_only ? 0 : e_port;
 755   m_key.protocol = addr_only ? 0 : proto;
 756   m_key.fib_index = 0;
 757   kv.key = m_key.as_u64;
 758   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 759     m = 0;
 760   else
 761     m = pool_elt_at_index (sm->static_mappings, value.value);
 762
 763   if (is_add)
 764     {
 765       if (m)
 766         {
 767           if (is_identity_static_mapping (m))
 768             {
 769               /* *INDENT-OFF* */
 770               pool_foreach (local, m->locals,
 771               ({
 772                 if (local->vrf_id == vrf_id)
 773                   return VNET_API_ERROR_VALUE_EXIST;
 774               }));
 775               /* *INDENT-ON* */
 776               pool_get (m->locals, local);
 777               local->vrf_id = vrf_id;
 778               local->fib_index =
 779                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 780                                                    FIB_SOURCE_PLUGIN_LOW);
 781               m_key.addr = m->local_addr;
 782               m_key.port = m->local_port;
 783               m_key.protocol = m->proto;
 784               m_key.fib_index = local->fib_index;
 785               kv.key = m_key.as_u64;
 786               kv.value = m - sm->static_mappings;
 787               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 788               return 0;
 789             }
 790           else
 791             return VNET_API_ERROR_VALUE_EXIST;
 792         }
 793
 794       if (twice_nat && addr_only)
 795         return VNET_API_ERROR_UNSUPPORTED;
 796
 797       /* Convert VRF id to FIB index */
 798       if (vrf_id != ~0)
 799         fib_index =
 800           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 801                                              FIB_SOURCE_PLUGIN_LOW);
 802       /* If not specified use inside VRF id from SNAT plugin startup config */
 803       else
 804         {
 805           fib_index = sm->inside_fib_index;
 806           vrf_id = sm->inside_vrf_id;
 807         }
 808
 809       if (!(out2in_only || identity_nat))
 810         {
 811           m_key.addr = l_addr;
 812           m_key.port = addr_only ? 0 : l_port;
 813           m_key.protocol = addr_only ? 0 : proto;
 814           m_key.fib_index = fib_index;
 815           kv.key = m_key.as_u64;
 816           if (!clib_bihash_search_8_8
 817               (&sm->static_mapping_by_local, &kv, &value))
 818             return VNET_API_ERROR_VALUE_EXIST;
 819         }
 820
 821       /* Find external address in allocated addresses and reserve port for
 822          address and port pair mapping when dynamic translations enabled */
 823       if (!(addr_only || sm->static_mapping_only || out2in_only))
 824         {
 825           for (i = 0; i < vec_len (sm->addresses); i++)
 826             {
 827               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 828                 {
 829                   a = sm->addresses + i;
 830                   /* External port must be unused */
 831                   switch (proto)
 832                     {
 833 #define _(N, j, n, s) \
 834                     case SNAT_PROTOCOL_##N: \
 835                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 836                         return VNET_API_ERROR_INVALID_VALUE; \
 837                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 838                       if (e_port > 1024) \
 839                         { \
 840                           a->busy_##n##_ports++; \
 841                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 842                         } \
 843                       break;
 844                       foreach_snat_protocol
 845 #undef _
 846                     default:
 847                       nat_log_info ("unknown protocol");
 848                       return VNET_API_ERROR_INVALID_VALUE_2;
 849                     }
 850                   break;
 851                 }
 852             }
 853           /* External address must be allocated */
 854           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 855             {
 856               if (sw_if_index != ~0)
 857                 {
 858                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 859                     {
 860                       rp = sm->to_resolve + i;
 861                       if (rp->addr_only)
 862                         continue;
 863                       if (rp->sw_if_index != sw_if_index &&
 864                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 865                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 866                           rp->e_port != e_port && rp->proto != proto)
 867                         continue;
 868
 869                       vec_del1 (sm->to_resolve, i);
 870                       break;
 871                     }
 872                 }
 873               return VNET_API_ERROR_NO_SUCH_ENTRY;
 874             }
 875         }
 876
 877       pool_get (sm->static_mappings, m);
 878       clib_memset (m, 0, sizeof (*m));
 879       m->tag = vec_dup (tag);
 880       m->local_addr = l_addr;
 881       m->external_addr = e_addr;
 882       m->twice_nat = twice_nat;
 883       if (out2in_only)
 884         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 885       if (addr_only)
 886         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 887       if (identity_nat)
 888         {
 889           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 890           pool_get (m->locals, local);
 891           local->vrf_id = vrf_id;
 892           local->fib_index = fib_index;
 893         }
 894       else
 895         {
 896           m->vrf_id = vrf_id;
 897           m->fib_index = fib_index;
 898         }
 899       if (!addr_only)
 900         {
 901           m->local_port = l_port;
 902           m->external_port = e_port;
 903           m->proto = proto;
 904         }
 905
 906       if (sm->num_workers > 1)
 907         {
 908           ip4_header_t ip = {
 909             .src_address = m->local_addr,
 910           };
 911           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
 912           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 913         }
 914       else
 915         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 916
 917       m_key.addr = m->local_addr;
 918       m_key.port = m->local_port;
 919       m_key.protocol = m->proto;
 920       m_key.fib_index = fib_index;
 921       kv.key = m_key.as_u64;
 922       kv.value = m - sm->static_mappings;
 923       if (!out2in_only)
 924         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 925
 926       m_key.addr = m->external_addr;
 927       m_key.port = m->external_port;
 928       m_key.fib_index = 0;
 929       kv.key = m_key.as_u64;
 930       kv.value = m - sm->static_mappings;
 931       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 932
 933       /* Delete dynamic sessions matching local address (+ local port) */
 934       if (!(sm->static_mapping_only))
 935         {
 936           u_key.addr = m->local_addr;
 937           u_key.fib_index = m->fib_index;
 938           kv.key = u_key.as_u64;
 939           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 940             {
 941               user_index = value.value;
 942               u = pool_elt_at_index (tsm->users, user_index);
 943               if (u->nsessions)
 944                 {
 945                   head_index = u->sessions_per_user_list_head_index;
 946                   head = pool_elt_at_index (tsm->list_pool, head_index);
 947                   elt_index = head->next;
 948                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 949                   ses_index = elt->value;
 950                   while (ses_index != ~0)
 951                     {
 952                       s = pool_elt_at_index (tsm->sessions, ses_index);
 953                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 954                       ses_index = elt->value;
 955
 956                       if (snat_is_session_static (s))
 957                         continue;
 958
 959                       if (!addr_only
 960                           && (clib_net_to_host_u16 (s->in2out.port) !=
 961                               m->local_port))
 962                         continue;
 963
 964                       nat_free_session_data (sm, s,
 965                                              tsm - sm->per_thread_data);
 966                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 967
 968                       if (!addr_only && !sm->endpoint_dependent)
 969                         break;
 970                     }
 971                 }
 972             }
 973         }
 974     }
 975   else
 976     {
 977       if (!m)
 978         {
 979           if (sw_if_index != ~0)
 980             return 0;
 981           else
 982             return VNET_API_ERROR_NO_SUCH_ENTRY;
 983         }
 984
 985       if (identity_nat)
 986         {
 987           if (vrf_id == ~0)
 988             vrf_id = sm->inside_vrf_id;
 989
 990           /* *INDENT-OFF* */
 991           pool_foreach (local, m->locals,
 992           ({
 993             if (local->vrf_id == vrf_id)
 994               find = local - m->locals;
 995           }));
 996           /* *INDENT-ON* */
 997           if (find == ~0)
 998             return VNET_API_ERROR_NO_SUCH_ENTRY;
 999
1000           local = pool_elt_at_index (m->locals, find);
1001           fib_index = local->fib_index;
1002           pool_put (m->locals, local);
1003         }
1004       else
1005         fib_index = m->fib_index;
1006
1007       /* Free external address port */
1008       if (!(addr_only || sm->static_mapping_only || out2in_only))
1009         {
1010           for (i = 0; i < vec_len (sm->addresses); i++)
1011             {
1012               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1013                 {
1014                   a = sm->addresses + i;
1015                   switch (proto)
1016                     {
1017 #define _(N, j, n, s) \
1018                     case SNAT_PROTOCOL_##N: \
1019                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1020                       if (e_port > 1024) \
1021                         { \
1022                           a->busy_##n##_ports--; \
1023                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1024                         } \
1025                       break;
1026                       foreach_snat_protocol
1027 #undef _
1028                     default:
1029                       nat_log_info ("unknown protocol");
1030                       return VNET_API_ERROR_INVALID_VALUE_2;
1031                     }
1032                   break;
1033                 }
1034             }
1035         }
1036
1037       if (sm->num_workers > 1)
1038         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1039       else
1040         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1041
1042       m_key.addr = m->local_addr;
1043       m_key.port = m->local_port;
1044       m_key.protocol = m->proto;
1045       m_key.fib_index = fib_index;
1046       kv.key = m_key.as_u64;
1047       if (!out2in_only)
1048         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1049
1050       /* Delete session(s) for static mapping if exist */
1051       if (!(sm->static_mapping_only) ||
1052           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1053         {
1054           u_key.addr = m->local_addr;
1055           u_key.fib_index = fib_index;
1056           kv.key = u_key.as_u64;
1057           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1058             {
1059               user_index = value.value;
1060               u = pool_elt_at_index (tsm->users, user_index);
1061               if (u->nstaticsessions)
1062                 {
1063                   head_index = u->sessions_per_user_list_head_index;
1064                   head = pool_elt_at_index (tsm->list_pool, head_index);
1065                   elt_index = head->next;
1066                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1067                   ses_index = elt->value;
1068                   while (ses_index != ~0)
1069                     {
1070                       s = pool_elt_at_index (tsm->sessions, ses_index);
1071                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1072                       ses_index = elt->value;
1073
1074                       if (!addr_only)
1075                         {
1076                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1077                               (clib_net_to_host_u16 (s->out2in.port) !=
1078                                e_port))
1079                             continue;
1080                         }
1081
1082                       if (is_lb_session (s))
1083                         continue;
1084
1085                       if (!snat_is_session_static (s))
1086                         continue;
1087
1088                       nat_free_session_data (sm, s,
1089                                              tsm - sm->per_thread_data);
1090                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1091
1092                       if (!addr_only && !sm->endpoint_dependent)
1093                         break;
1094                     }
1095                 }
1096             }
1097         }
1098
1099       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1100       if (pool_elts (m->locals))
1101         return 0;
1102
1103       m_key.addr = m->external_addr;
1104       m_key.port = m->external_port;
1105       m_key.fib_index = 0;
1106       kv.key = m_key.as_u64;
1107       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1108
1109       vec_free (m->tag);
1110       vec_free (m->workers);
1111       /* Delete static mapping from pool */
1112       pool_put (sm->static_mappings, m);
1113     }
1114
1115   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1116     return 0;
1117
1118   /* Add/delete external address to FIB */
1119   /* *INDENT-OFF* */
1120   pool_foreach (interface, sm->interfaces,
1121   ({
1122     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1123       continue;
1124
1125     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1126     break;
1127   }));
1128   pool_foreach (interface, sm->output_feature_interfaces,
1129   ({
1130     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1131       continue;
1132
1133     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1134     break;
1135   }));
1136   /* *INDENT-ON* */
1137
1138   return 0;
1139 }
1140
1141 int
1142 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1143                                  snat_protocol_t proto,
1144                                  nat44_lb_addr_port_t * locals, u8 is_add,
1145                                  twice_nat_type_t twice_nat, u8 out2in_only,
1146                                  u8 * tag, u32 affinity)
1147 {
1148   snat_main_t *sm = &snat_main;
1149   snat_static_mapping_t *m;
1150   snat_session_key_t m_key;
1151   clib_bihash_kv_8_8_t kv, value;
1152   snat_address_t *a = 0;
1153   int i;
1154   nat44_lb_addr_port_t *local;
1155   u32 elt_index, head_index, ses_index;
1156   snat_main_per_thread_data_t *tsm;
1157   snat_user_key_t u_key;
1158   snat_user_t *u;
1159   snat_session_t *s;
1160   dlist_elt_t *head, *elt;
1161   uword *bitmap = 0;
1162
1163   if (!sm->endpoint_dependent)
1164     return VNET_API_ERROR_FEATURE_DISABLED;
1165
1166   m_key.addr = e_addr;
1167   m_key.port = e_port;
1168   m_key.protocol = proto;
1169   m_key.fib_index = 0;
1170   kv.key = m_key.as_u64;
1171   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1172     m = 0;
1173   else
1174     m = pool_elt_at_index (sm->static_mappings, value.value);
1175
1176   if (is_add)
1177     {
1178       if (m)
1179         return VNET_API_ERROR_VALUE_EXIST;
1180
1181       if (vec_len (locals) < 2)
1182         return VNET_API_ERROR_INVALID_VALUE;
1183
1184       /* Find external address in allocated addresses and reserve port for
1185          address and port pair mapping when dynamic translations enabled */
1186       if (!(sm->static_mapping_only || out2in_only))
1187         {
1188           for (i = 0; i < vec_len (sm->addresses); i++)
1189             {
1190               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1191                 {
1192                   a = sm->addresses + i;
1193                   /* External port must be unused */
1194                   switch (proto)
1195                     {
1196 #define _(N, j, n, s) \
1197                     case SNAT_PROTOCOL_##N: \
1198                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1199                         return VNET_API_ERROR_INVALID_VALUE; \
1200                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1201                       if (e_port > 1024) \
1202                         { \
1203                           a->busy_##n##_ports++; \
1204                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1205                         } \
1206                       break;
1207                       foreach_snat_protocol
1208 #undef _
1209                     default:
1210                       nat_log_info ("unknown protocol");
1211                       return VNET_API_ERROR_INVALID_VALUE_2;
1212                     }
1213                   break;
1214                 }
1215             }
1216           /* External address must be allocated */
1217           if (!a)
1218             return VNET_API_ERROR_NO_SUCH_ENTRY;
1219         }
1220
1221       pool_get (sm->static_mappings, m);
1222       clib_memset (m, 0, sizeof (*m));
1223       m->tag = vec_dup (tag);
1224       m->external_addr = e_addr;
1225       m->external_port = e_port;
1226       m->proto = proto;
1227       m->twice_nat = twice_nat;
1228       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1229       if (out2in_only)
1230         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1231       m->affinity = affinity;
1232
1233       if (affinity)
1234         m->affinity_per_service_list_head_index =
1235           nat_affinity_get_per_service_list_head_index ();
1236       else
1237         m->affinity_per_service_list_head_index = ~0;
1238
1239       m_key.addr = m->external_addr;
1240       m_key.port = m->external_port;
1241       m_key.protocol = m->proto;
1242       m_key.fib_index = 0;
1243       kv.key = m_key.as_u64;
1244       kv.value = m - sm->static_mappings;
1245       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1246         {
1247           nat_log_err ("static_mapping_by_external key add failed");
1248           return VNET_API_ERROR_UNSPECIFIED;
1249         }
1250
1251       m_key.fib_index = m->fib_index;
1252       for (i = 0; i < vec_len (locals); i++)
1253         {
1254           locals[i].fib_index =
1255             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1256                                                locals[i].vrf_id,
1257                                                FIB_SOURCE_PLUGIN_LOW);
1258           m_key.addr = locals[i].addr;
1259           m_key.fib_index = locals[i].fib_index;
1260           if (!out2in_only)
1261             {
1262               m_key.port = locals[i].port;
1263               kv.key = m_key.as_u64;
1264               kv.value = m - sm->static_mappings;
1265               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1266             }
1267           locals[i].prefix = (i == 0) ? locals[i].probability :
1268             (locals[i - 1].prefix + locals[i].probability);
1269           pool_get (m->locals, local);
1270           *local = locals[i];
1271           if (sm->num_workers > 1)
1272             {
1273               ip4_header_t ip = {
1274                 .src_address = locals[i].addr,
1275               };
1276               bitmap =
1277                 clib_bitmap_set (bitmap,
1278                                  sm->worker_in2out_cb (&ip, m->fib_index), 1);
1279             }
1280         }
1281
1282       /* Assign workers */
1283       if (sm->num_workers > 1)
1284         {
1285           /* *INDENT-OFF* */
1286           clib_bitmap_foreach (i, bitmap,
1287             ({
1288                vec_add1(m->workers, i);
1289             }));
1290           /* *INDENT-ON* */
1291         }
1292     }
1293   else
1294     {
1295       if (!m)
1296         return VNET_API_ERROR_NO_SUCH_ENTRY;
1297
1298       if (!is_lb_static_mapping (m))
1299         return VNET_API_ERROR_INVALID_VALUE;
1300
1301       /* Free external address port */
1302       if (!(sm->static_mapping_only || out2in_only))
1303         {
1304           for (i = 0; i < vec_len (sm->addresses); i++)
1305             {
1306               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1307                 {
1308                   a = sm->addresses + i;
1309                   switch (proto)
1310                     {
1311 #define _(N, j, n, s) \
1312                     case SNAT_PROTOCOL_##N: \
1313                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1314                       if (e_port > 1024) \
1315                         { \
1316                           a->busy_##n##_ports--; \
1317                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1318                         } \
1319                       break;
1320                       foreach_snat_protocol
1321 #undef _
1322                     default:
1323                       nat_log_info ("unknown protocol");
1324                       return VNET_API_ERROR_INVALID_VALUE_2;
1325                     }
1326                   break;
1327                 }
1328             }
1329         }
1330
1331       m_key.addr = m->external_addr;
1332       m_key.port = m->external_port;
1333       m_key.protocol = m->proto;
1334       m_key.fib_index = 0;
1335       kv.key = m_key.as_u64;
1336       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1337         {
1338           nat_log_err ("static_mapping_by_external key del failed");
1339           return VNET_API_ERROR_UNSPECIFIED;
1340         }
1341
1342       /* *INDENT-OFF* */
1343       pool_foreach (local, m->locals,
1344       ({
1345           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1346                             FIB_SOURCE_PLUGIN_LOW);
1347           m_key.addr = local->addr;
1348           if (!out2in_only)
1349             {
1350               m_key.port = local->port;
1351               m_key.fib_index = local->fib_index;
1352               kv.key = m_key.as_u64;
1353               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1354                 {
1355                   nat_log_err ("static_mapping_by_local key del failed");
1356                   return VNET_API_ERROR_UNSPECIFIED;
1357                 }
1358             }
1359
1360           if (sm->num_workers > 1)
1361             {
1362               ip4_header_t ip = {
1363                 .src_address = local->addr,
1364               };
1365               tsm = vec_elt_at_index (sm->per_thread_data,
1366                                       sm->worker_in2out_cb (&ip, m->fib_index));
1367             }
1368           else
1369             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1370
1371           /* Delete sessions */
1372           u_key.addr = local->addr;
1373           u_key.fib_index = local->fib_index;
1374           kv.key = u_key.as_u64;
1375           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1376             {
1377               u = pool_elt_at_index (tsm->users, value.value);
1378               if (u->nstaticsessions)
1379                 {
1380                   head_index = u->sessions_per_user_list_head_index;
1381                   head = pool_elt_at_index (tsm->list_pool, head_index);
1382                   elt_index = head->next;
1383                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1384                   ses_index = elt->value;
1385                   while (ses_index != ~0)
1386                     {
1387                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1388                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1389                       ses_index = elt->value;
1390
1391                       if (!(is_lb_session (s)))
1392                         continue;
1393
1394                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1395                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1396                         continue;
1397
1398                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1399                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1400                     }
1401                 }
1402             }
1403       }));
1404       /* *INDENT-ON* */
1405       if (m->affinity)
1406         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1407       pool_free (m->locals);
1408       vec_free (m->tag);
1409       vec_free (m->workers);
1410
1411       pool_put (sm->static_mappings, m);
1412     }
1413
1414   return 0;
1415 }
1416
1417 int
1418 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1419                                        ip4_address_t l_addr, u16 l_port,
1420                                        snat_protocol_t proto, u32 vrf_id,
1421                                        u8 probability, u8 is_add)
1422 {
1423   snat_main_t *sm = &snat_main;
1424   snat_static_mapping_t *m = 0;
1425   snat_session_key_t m_key;
1426   clib_bihash_kv_8_8_t kv, value;
1427   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1428   snat_main_per_thread_data_t *tsm;
1429   snat_user_key_t u_key;
1430   snat_user_t *u;
1431   snat_session_t *s;
1432   dlist_elt_t *head, *elt;
1433   u32 elt_index, head_index, ses_index, *locals = 0;
1434   uword *bitmap = 0;
1435   int i;
1436
1437   if (!sm->endpoint_dependent)
1438     return VNET_API_ERROR_FEATURE_DISABLED;
1439
1440   m_key.addr = e_addr;
1441   m_key.port = e_port;
1442   m_key.protocol = proto;
1443   m_key.fib_index = 0;
1444   kv.key = m_key.as_u64;
1445   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1446     m = pool_elt_at_index (sm->static_mappings, value.value);
1447
1448   if (!m)
1449     return VNET_API_ERROR_NO_SUCH_ENTRY;
1450
1451   if (!is_lb_static_mapping (m))
1452     return VNET_API_ERROR_INVALID_VALUE;
1453
1454   /* *INDENT-OFF* */
1455   pool_foreach (local, m->locals,
1456   ({
1457     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1458         (local->vrf_id == vrf_id))
1459       {
1460         match_local = local;
1461         break;
1462       }
1463   }));
1464   /* *INDENT-ON* */
1465
1466   if (is_add)
1467     {
1468       if (match_local)
1469         return VNET_API_ERROR_VALUE_EXIST;
1470
1471       pool_get (m->locals, local);
1472       clib_memset (local, 0, sizeof (*local));
1473       local->addr.as_u32 = l_addr.as_u32;
1474       local->port = l_port;
1475       local->probability = probability;
1476       local->vrf_id = vrf_id;
1477       local->fib_index =
1478         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1479                                            FIB_SOURCE_PLUGIN_LOW);
1480
1481       if (!is_out2in_only_static_mapping (m))
1482         {
1483           m_key.addr = l_addr;
1484           m_key.port = l_port;
1485           m_key.fib_index = local->fib_index;
1486           kv.key = m_key.as_u64;
1487           kv.value = m - sm->static_mappings;
1488           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1489             nat_log_err ("static_mapping_by_local key add failed");
1490         }
1491     }
1492   else
1493     {
1494       if (!match_local)
1495         return VNET_API_ERROR_NO_SUCH_ENTRY;
1496
1497       if (pool_elts (m->locals) < 3)
1498         return VNET_API_ERROR_UNSPECIFIED;
1499
1500       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1501                         FIB_SOURCE_PLUGIN_LOW);
1502
1503       if (!is_out2in_only_static_mapping (m))
1504         {
1505           m_key.addr = l_addr;
1506           m_key.port = l_port;
1507           m_key.fib_index = match_local->fib_index;
1508           kv.key = m_key.as_u64;
1509           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1510             nat_log_err ("static_mapping_by_local key del failed");
1511         }
1512
1513       if (sm->num_workers > 1)
1514         {
1515           ip4_header_t ip = {
1516             .src_address = local->addr,
1517           };
1518           tsm = vec_elt_at_index (sm->per_thread_data,
1519                                   sm->worker_in2out_cb (&ip, m->fib_index));
1520         }
1521       else
1522         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1523
1524       /* Delete sessions */
1525       u_key.addr = match_local->addr;
1526       u_key.fib_index = match_local->fib_index;
1527       kv.key = u_key.as_u64;
1528       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1529         {
1530           u = pool_elt_at_index (tsm->users, value.value);
1531           if (u->nstaticsessions)
1532             {
1533               head_index = u->sessions_per_user_list_head_index;
1534               head = pool_elt_at_index (tsm->list_pool, head_index);
1535               elt_index = head->next;
1536               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1537               ses_index = elt->value;
1538               while (ses_index != ~0)
1539                 {
1540                   s = pool_elt_at_index (tsm->sessions, ses_index);
1541                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1542                   ses_index = elt->value;
1543
1544                   if (!(is_lb_session (s)))
1545                     continue;
1546
1547                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1548                       (clib_net_to_host_u16 (s->in2out.port) !=
1549                        match_local->port))
1550                     continue;
1551
1552                   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1553                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1554                 }
1555             }
1556         }
1557
1558       pool_put (m->locals, match_local);
1559     }
1560
1561   vec_free (m->workers);
1562
1563   /* *INDENT-OFF* */
1564   pool_foreach (local, m->locals,
1565   ({
1566     vec_add1 (locals, local - m->locals);
1567     if (sm->num_workers > 1)
1568       {
1569         ip4_header_t ip;
1570         ip.src_address.as_u32 = local->addr.as_u32,
1571         bitmap = clib_bitmap_set (bitmap,
1572                                   sm->worker_in2out_cb (&ip, local->fib_index),
1573                                   1);
1574       }
1575   }));
1576   /* *INDENT-ON* */
1577
1578   local = pool_elt_at_index (m->locals, locals[0]);
1579   local->prefix = local->probability;
1580   for (i = 1; i < vec_len (locals); i++)
1581     {
1582       local = pool_elt_at_index (m->locals, locals[i]);
1583       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1584       local->prefix = local->probability + prev_local->prefix;
1585     }
1586
1587   /* Assign workers */
1588   if (sm->num_workers > 1)
1589     {
1590       /* *INDENT-OFF* */
1591       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1592       /* *INDENT-ON* */
1593     }
1594
1595   return 0;
1596 }
1597
1598 int
1599 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1600                   u8 twice_nat)
1601 {
1602   snat_address_t *a = 0;
1603   snat_session_t *ses;
1604   u32 *ses_to_be_removed = 0, *ses_index;
1605   snat_main_per_thread_data_t *tsm;
1606   snat_static_mapping_t *m;
1607   snat_interface_t *interface;
1608   int i;
1609   snat_address_t *addresses =
1610     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1611
1612   /* Find SNAT address */
1613   for (i = 0; i < vec_len (addresses); i++)
1614     {
1615       if (addresses[i].addr.as_u32 == addr.as_u32)
1616         {
1617           a = addresses + i;
1618           break;
1619         }
1620     }
1621   if (!a)
1622     return VNET_API_ERROR_NO_SUCH_ENTRY;
1623
1624   if (delete_sm)
1625     {
1626       /* *INDENT-OFF* */
1627       pool_foreach (m, sm->static_mappings,
1628       ({
1629           if (m->external_addr.as_u32 == addr.as_u32)
1630             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1631                                             m->local_port, m->external_port,
1632                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1633                                             m->proto, 0, m->twice_nat,
1634                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1635       }));
1636       /* *INDENT-ON* */
1637     }
1638   else
1639     {
1640       /* Check if address is used in some static mapping */
1641       if (is_snat_address_used_in_static_mapping (sm, addr))
1642         {
1643           nat_log_notice ("address used in static mapping");
1644           return VNET_API_ERROR_UNSPECIFIED;
1645         }
1646     }
1647
1648   if (a->fib_index != ~0)
1649     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1650
1651   /* Delete sessions using address */
1652   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1653     {
1654       /* *INDENT-OFF* */
1655       vec_foreach (tsm, sm->per_thread_data)
1656         {
1657           pool_foreach (ses, tsm->sessions, ({
1658             if (ses->out2in.addr.as_u32 == addr.as_u32)
1659               {
1660                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1661                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1662               }
1663           }));
1664
1665           vec_foreach (ses_index, ses_to_be_removed)
1666             {
1667               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1668               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1669             }
1670
1671           vec_free (ses_to_be_removed);
1672         }
1673       /* *INDENT-ON* */
1674     }
1675
1676 #define _(N, i, n, s) \
1677   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1678   vec_free (a->busy_##n##_ports_per_thread);
1679   foreach_snat_protocol
1680 #undef _
1681     if (twice_nat)
1682     {
1683       vec_del1 (sm->twice_nat_addresses, i);
1684       return 0;
1685     }
1686   else
1687     vec_del1 (sm->addresses, i);
1688
1689   /* Delete external address from FIB */
1690   /* *INDENT-OFF* */
1691   pool_foreach (interface, sm->interfaces,
1692   ({
1693     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1694       continue;
1695
1696     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1697     break;
1698   }));
1699   pool_foreach (interface, sm->output_feature_interfaces,
1700   ({
1701     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1702       continue;
1703
1704     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1705     break;
1706   }));
1707   /* *INDENT-ON* */
1708
1709   return 0;
1710 }
1711
1712 int
1713 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1714 {
1715   snat_main_t *sm = &snat_main;
1716   snat_interface_t *i;
1717   const char *feature_name, *del_feature_name;
1718   snat_address_t *ap;
1719   snat_static_mapping_t *m;
1720   snat_det_map_t *dm;
1721   nat_outside_fib_t *outside_fib;
1722   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1723                                                        sw_if_index);
1724
1725   if (sm->out2in_dpo && !is_inside)
1726     return VNET_API_ERROR_UNSUPPORTED;
1727
1728   /* *INDENT-OFF* */
1729   pool_foreach (i, sm->output_feature_interfaces,
1730   ({
1731     if (i->sw_if_index == sw_if_index)
1732       return VNET_API_ERROR_VALUE_EXIST;
1733   }));
1734   /* *INDENT-ON* */
1735
1736   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1737     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1738   else
1739     {
1740       if (sm->num_workers > 1 && !sm->deterministic)
1741         feature_name =
1742           is_inside ? "nat44-in2out-worker-handoff" :
1743           "nat44-out2in-worker-handoff";
1744       else if (sm->deterministic)
1745         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1746       else if (sm->endpoint_dependent)
1747         feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
1748       else
1749         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1750     }
1751
1752   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1753     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1754                                                       NAT_FQ_NELTS);
1755
1756   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1757     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1758                                                       NAT_FQ_NELTS);
1759
1760   if (!is_inside)
1761     {
1762       /* *INDENT-OFF* */
1763       vec_foreach (outside_fib, sm->outside_fibs)
1764         {
1765           if (outside_fib->fib_index == fib_index)
1766             {
1767               if (is_del)
1768                 {
1769                   outside_fib->refcount--;
1770                   if (!outside_fib->refcount)
1771                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1772                 }
1773               else
1774                 outside_fib->refcount++;
1775               goto feature_set;
1776             }
1777         }
1778       /* *INDENT-ON* */
1779       if (!is_del)
1780         {
1781           vec_add2 (sm->outside_fibs, outside_fib, 1);
1782           outside_fib->refcount = 1;
1783           outside_fib->fib_index = fib_index;
1784         }
1785     }
1786 feature_set:
1787   /* *INDENT-OFF* */
1788   pool_foreach (i, sm->interfaces,
1789   ({
1790     if (i->sw_if_index == sw_if_index)
1791       {
1792         if (is_del)
1793           {
1794             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1795               {
1796                 if (is_inside)
1797                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1798                 else
1799                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1800
1801                 if (sm->num_workers > 1 && !sm->deterministic)
1802                   {
1803                     del_feature_name = "nat44-handoff-classify";
1804                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1805                                                  "nat44-out2in-worker-handoff";
1806                   }
1807                 else if (sm->deterministic)
1808                   {
1809                     del_feature_name = "nat44-det-classify";
1810                     feature_name = !is_inside ?  "nat44-det-in2out" :
1811                                                  "nat44-det-out2in";
1812                   }
1813                 else if (sm->endpoint_dependent)
1814                   {
1815                     del_feature_name = "nat44-ed-classify";
1816                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1817                                                  "nat44-ed-out2in";
1818                   }
1819                 else
1820                   {
1821                     del_feature_name = "nat44-classify";
1822                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1823                   }
1824
1825                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1826                                              sw_if_index, 0, 0, 0);
1827                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1828                                              sw_if_index, 1, 0, 0);
1829                 if (!is_inside)
1830                   {
1831                     if (sm->endpoint_dependent)
1832                       vnet_feature_enable_disable ("ip4-local",
1833                                                    "nat44-ed-hairpinning",
1834                                                    sw_if_index, 1, 0, 0);
1835                     else if (!sm->deterministic)
1836                       vnet_feature_enable_disable ("ip4-local",
1837                                                    "nat44-hairpinning",
1838                                                    sw_if_index, 1, 0, 0);
1839                   }
1840               }
1841             else
1842               {
1843                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1844                                              sw_if_index, 0, 0, 0);
1845                 pool_put (sm->interfaces, i);
1846                 if (is_inside)
1847                   {
1848                     if (sm->endpoint_dependent)
1849                       vnet_feature_enable_disable ("ip4-local",
1850                                                    "nat44-ed-hairpinning",
1851                                                    sw_if_index, 0, 0, 0);
1852                     else if (!sm->deterministic)
1853                       vnet_feature_enable_disable ("ip4-local",
1854                                                    "nat44-hairpinning",
1855                                                    sw_if_index, 0, 0, 0);
1856                   }
1857               }
1858           }
1859         else
1860           {
1861             if ((nat_interface_is_inside(i) && is_inside) ||
1862                 (nat_interface_is_outside(i) && !is_inside))
1863               return 0;
1864
1865             if (sm->num_workers > 1 && !sm->deterministic)
1866               {
1867                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1868                                                  "nat44-out2in-worker-handoff";
1869                 feature_name = "nat44-handoff-classify";
1870               }
1871             else if (sm->deterministic)
1872               {
1873                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1874                                                  "nat44-det-out2in";
1875                 feature_name = "nat44-det-classify";
1876               }
1877             else if (sm->endpoint_dependent)
1878               {
1879                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1880                                                  "nat44-ed-out2in";
1881                 feature_name = "nat44-ed-classify";
1882               }
1883             else
1884               {
1885                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1886                 feature_name = "nat44-classify";
1887               }
1888
1889             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1890                                          sw_if_index, 0, 0, 0);
1891             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1892                                          sw_if_index, 1, 0, 0);
1893             if (!is_inside)
1894               {
1895                 if (sm->endpoint_dependent)
1896                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1897                                                sw_if_index, 0, 0, 0);
1898                 else if (!sm->deterministic)
1899                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1900                                                sw_if_index, 0, 0, 0);
1901               }
1902             goto set_flags;
1903           }
1904
1905         goto fib;
1906       }
1907   }));
1908   /* *INDENT-ON* */
1909
1910   if (is_del)
1911     return VNET_API_ERROR_NO_SUCH_ENTRY;
1912
1913   pool_get (sm->interfaces, i);
1914   i->sw_if_index = sw_if_index;
1915   i->flags = 0;
1916   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1917                                0);
1918
1919   if (is_inside && !sm->out2in_dpo)
1920     {
1921       if (sm->endpoint_dependent)
1922         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1923                                      sw_if_index, 1, 0, 0);
1924       else if (!sm->deterministic)
1925         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1926                                      sw_if_index, 1, 0, 0);
1927     }
1928
1929 set_flags:
1930   if (is_inside)
1931     {
1932       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1933       return 0;
1934     }
1935   else
1936     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1937
1938   /* Add/delete external addresses to FIB */
1939 fib:
1940   /* *INDENT-OFF* */
1941   vec_foreach (ap, sm->addresses)
1942     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1943
1944   pool_foreach (m, sm->static_mappings,
1945   ({
1946     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1947       continue;
1948
1949     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1950   }));
1951
1952   pool_foreach (dm, sm->det_maps,
1953   ({
1954     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1955   }));
1956   /* *INDENT-ON* */
1957
1958   return 0;
1959 }
1960
1961 int
1962 snat_interface_add_del_output_feature (u32 sw_if_index,
1963                                        u8 is_inside, int is_del)
1964 {
1965   snat_main_t *sm = &snat_main;
1966   snat_interface_t *i;
1967   snat_address_t *ap;
1968   snat_static_mapping_t *m;
1969
1970   if (sm->deterministic ||
1971       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1972     return VNET_API_ERROR_UNSUPPORTED;
1973
1974   /* *INDENT-OFF* */
1975   pool_foreach (i, sm->interfaces,
1976   ({
1977     if (i->sw_if_index == sw_if_index)
1978       return VNET_API_ERROR_VALUE_EXIST;
1979   }));
1980   /* *INDENT-ON* */
1981
1982   if (is_inside)
1983     {
1984       if (sm->endpoint_dependent)
1985         {
1986           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
1987                                        sw_if_index, !is_del, 0, 0);
1988           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
1989                                        sw_if_index, !is_del, 0, 0);
1990         }
1991       else
1992         {
1993           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1994                                        sw_if_index, !is_del, 0, 0);
1995           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1996                                        sw_if_index, !is_del, 0, 0);
1997         }
1998       goto fq;
1999     }
2000
2001   if (sm->num_workers > 1)
2002     {
2003       vnet_feature_enable_disable ("ip4-unicast",
2004                                    "nat44-out2in-worker-handoff",
2005                                    sw_if_index, !is_del, 0, 0);
2006       vnet_feature_enable_disable ("ip4-output",
2007                                    "nat44-in2out-output-worker-handoff",
2008                                    sw_if_index, !is_del, 0, 0);
2009     }
2010   else
2011     {
2012       if (sm->endpoint_dependent)
2013         {
2014           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
2015                                        sw_if_index, !is_del, 0, 0);
2016           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2017                                        sw_if_index, !is_del, 0, 0);
2018         }
2019       else
2020         {
2021           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2022                                        sw_if_index, !is_del, 0, 0);
2023           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2024                                        sw_if_index, !is_del, 0, 0);
2025         }
2026     }
2027
2028 fq:
2029   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2030     sm->fq_in2out_output_index =
2031       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2032
2033   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2034     sm->fq_out2in_index =
2035       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2036
2037   /* *INDENT-OFF* */
2038   pool_foreach (i, sm->output_feature_interfaces,
2039   ({
2040     if (i->sw_if_index == sw_if_index)
2041       {
2042         if (is_del)
2043           pool_put (sm->output_feature_interfaces, i);
2044         else
2045           return VNET_API_ERROR_VALUE_EXIST;
2046
2047         goto fib;
2048       }
2049   }));
2050   /* *INDENT-ON* */
2051
2052   if (is_del)
2053     return VNET_API_ERROR_NO_SUCH_ENTRY;
2054
2055   pool_get (sm->output_feature_interfaces, i);
2056   i->sw_if_index = sw_if_index;
2057   i->flags = 0;
2058   if (is_inside)
2059     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2060   else
2061     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2062
2063   /* Add/delete external addresses to FIB */
2064 fib:
2065   if (is_inside)
2066     return 0;
2067
2068   /* *INDENT-OFF* */
2069   vec_foreach (ap, sm->addresses)
2070     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2071
2072   pool_foreach (m, sm->static_mappings,
2073   ({
2074     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2075       continue;
2076
2077     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2078   }));
2079   /* *INDENT-ON* */
2080
2081   return 0;
2082 }
2083
2084 int
2085 snat_set_workers (uword * bitmap)
2086 {
2087   snat_main_t *sm = &snat_main;
2088   int i, j = 0;
2089
2090   if (sm->num_workers < 2)
2091     return VNET_API_ERROR_FEATURE_DISABLED;
2092
2093   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2094     return VNET_API_ERROR_INVALID_WORKER;
2095
2096   vec_free (sm->workers);
2097   /* *INDENT-OFF* */
2098   clib_bitmap_foreach (i, bitmap,
2099     ({
2100       vec_add1(sm->workers, i);
2101       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2102       j++;
2103     }));
2104   /* *INDENT-ON* */
2105
2106   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2107   sm->num_snat_thread = _vec_len (sm->workers);
2108
2109   return 0;
2110 }
2111
2112
2113 static void
2114 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2115                                        uword opaque,
2116                                        u32 sw_if_index,
2117                                        ip4_address_t * address,
2118                                        u32 address_length,
2119                                        u32 if_address_index, u32 is_delete);
2120
2121 static void
2122 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2123                                  uword opaque,
2124                                  u32 sw_if_index,
2125                                  ip4_address_t * address,
2126                                  u32 address_length,
2127                                  u32 if_address_index, u32 is_delete);
2128
2129 static int
2130 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2131                                  u32 fib_index,
2132                                  u32 thread_index,
2133                                  snat_session_key_t * k,
2134                                  u16 port_per_thread, u32 snat_thread_index);
2135
2136 static clib_error_t *
2137 snat_init (vlib_main_t * vm)
2138 {
2139   snat_main_t *sm = &snat_main;
2140   clib_error_t *error = 0;
2141   ip4_main_t *im = &ip4_main;
2142   ip_lookup_main_t *lm = &im->lookup_main;
2143   uword *p;
2144   vlib_thread_registration_t *tr;
2145   vlib_thread_main_t *tm = vlib_get_thread_main ();
2146   uword *bitmap = 0;
2147   u32 i;
2148   ip4_add_del_interface_address_callback_t cb4;
2149   vlib_node_t *error_drop_node;
2150
2151   sm->vlib_main = vm;
2152   sm->vnet_main = vnet_get_main ();
2153   sm->ip4_main = im;
2154   sm->ip4_lookup_main = lm;
2155   sm->api_main = &api_main;
2156   sm->first_worker_index = 0;
2157   sm->num_workers = 0;
2158   sm->num_snat_thread = 1;
2159   sm->workers = 0;
2160   sm->port_per_thread = 0xffff - 1024;
2161   sm->fq_in2out_index = ~0;
2162   sm->fq_out2in_index = ~0;
2163   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2164   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2165   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2166   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2167   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2168   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2169   sm->forwarding_enabled = 0;
2170   sm->log_class = vlib_log_register_class ("nat", 0);
2171   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2172   sm->error_node_index = error_drop_node->index;
2173   sm->mss_clamping = 0;
2174
2175   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2176   if (p)
2177     {
2178       tr = (vlib_thread_registration_t *) p[0];
2179       if (tr)
2180         {
2181           sm->num_workers = tr->count;
2182           sm->first_worker_index = tr->first_index;
2183         }
2184     }
2185
2186   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2187
2188   /* Use all available workers by default */
2189   if (sm->num_workers > 1)
2190     {
2191       for (i = 0; i < sm->num_workers; i++)
2192         bitmap = clib_bitmap_set (bitmap, i, 1);
2193       snat_set_workers (bitmap);
2194       clib_bitmap_free (bitmap);
2195     }
2196   else
2197     {
2198       sm->per_thread_data[0].snat_thread_index = 0;
2199     }
2200
2201   error = snat_api_init (vm, sm);
2202   if (error)
2203     return error;
2204
2205   /* Set up the interface address add/del callback */
2206   cb4.function = snat_ip4_add_del_interface_address_cb;
2207   cb4.function_opaque = 0;
2208
2209   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2210
2211   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2212   cb4.function_opaque = 0;
2213
2214   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2215
2216   nat_dpo_module_init ();
2217
2218   /* Init counters */
2219   sm->total_users.name = "total-users";
2220   sm->total_users.stat_segment_name = "/nat44/total-users";
2221   vlib_validate_simple_counter (&sm->total_users, 0);
2222   vlib_zero_simple_counter (&sm->total_users, 0);
2223   sm->total_sessions.name = "total-sessions";
2224   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2225   vlib_validate_simple_counter (&sm->total_sessions, 0);
2226   vlib_zero_simple_counter (&sm->total_sessions, 0);
2227
2228   /* Init IPFIX logging */
2229   snat_ipfix_logging_init (vm);
2230
2231   /* Init NAT64 */
2232   error = nat64_init (vm);
2233   if (error)
2234     return error;
2235
2236   dslite_init (vm);
2237
2238   nat66_init ();
2239
2240   /* Init virtual fragmenentation reassembly */
2241   return nat_reass_init (vm);
2242 }
2243
2244 VLIB_INIT_FUNCTION (snat_init);
2245
2246 void
2247 snat_free_outside_address_and_port (snat_address_t * addresses,
2248                                     u32 thread_index, snat_session_key_t * k)
2249 {
2250   snat_address_t *a;
2251   u32 address_index;
2252   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2253
2254   for (address_index = 0; address_index < vec_len (addresses);
2255        address_index++)
2256     {
2257       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2258         break;
2259     }
2260
2261   ASSERT (address_index < vec_len (addresses));
2262
2263   a = addresses + address_index;
2264
2265   switch (k->protocol)
2266     {
2267 #define _(N, i, n, s) \
2268     case SNAT_PROTOCOL_##N: \
2269       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2270         port_host_byte_order) == 1); \
2271       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2272         port_host_byte_order, 0); \
2273       a->busy_##n##_ports--; \
2274       a->busy_##n##_ports_per_thread[thread_index]--; \
2275       break;
2276       foreach_snat_protocol
2277 #undef _
2278     default:
2279       nat_log_info ("unknown protocol");
2280       return;
2281     }
2282 }
2283
2284 int
2285 snat_static_mapping_match (snat_main_t * sm,
2286                            snat_session_key_t match,
2287                            snat_session_key_t * mapping,
2288                            u8 by_external,
2289                            u8 * is_addr_only,
2290                            twice_nat_type_t * twice_nat,
2291                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2292                            u8 * is_identity_nat)
2293 {
2294   clib_bihash_kv_8_8_t kv, value;
2295   snat_static_mapping_t *m;
2296   snat_session_key_t m_key;
2297   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2298   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2299   u8 backend_index;
2300   nat44_lb_addr_port_t *local;
2301
2302   m_key.fib_index = match.fib_index;
2303   if (by_external)
2304     {
2305       mapping_hash = &sm->static_mapping_by_external;
2306       m_key.fib_index = 0;
2307     }
2308
2309   m_key.addr = match.addr;
2310   m_key.port = clib_net_to_host_u16 (match.port);
2311   m_key.protocol = match.protocol;
2312
2313   kv.key = m_key.as_u64;
2314
2315   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2316     {
2317       /* Try address only mapping */
2318       m_key.port = 0;
2319       m_key.protocol = 0;
2320       kv.key = m_key.as_u64;
2321       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2322         return 1;
2323     }
2324
2325   m = pool_elt_at_index (sm->static_mappings, value.value);
2326
2327   if (by_external)
2328     {
2329       if (is_lb_static_mapping (m))
2330         {
2331           if (PREDICT_FALSE (lb != 0))
2332             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2333           if (m->affinity)
2334             {
2335               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2336                                               match.protocol, match.port,
2337                                               &backend_index))
2338                 goto get_local;
2339
2340               local = pool_elt_at_index (m->locals, backend_index);
2341               mapping->addr = local->addr;
2342               mapping->port = clib_host_to_net_u16 (local->port);
2343               mapping->fib_index = local->fib_index;
2344               goto end;
2345             }
2346         get_local:
2347           /* *INDENT-OFF* */
2348           pool_foreach_index (i, m->locals,
2349           ({
2350             vec_add1 (tmp, i);
2351           }));
2352           /* *INDENT-ON* */
2353           hi = vec_len (tmp) - 1;
2354           local = pool_elt_at_index (m->locals, tmp[hi]);
2355           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2356           while (lo < hi)
2357             {
2358               mid = ((hi - lo) >> 1) + lo;
2359               local = pool_elt_at_index (m->locals, tmp[mid]);
2360               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2361             }
2362           local = pool_elt_at_index (m->locals, tmp[lo]);
2363           if (!(local->prefix >= rand))
2364             return 1;
2365           if (PREDICT_FALSE (sm->num_workers > 1))
2366             {
2367               ip4_header_t ip = {
2368                 .src_address = local->addr,
2369               };
2370               if (sm->worker_in2out_cb (&ip, m->fib_index) !=
2371                   vlib_get_thread_index ())
2372                 goto get_local;
2373             }
2374           mapping->addr = local->addr;
2375           mapping->port = clib_host_to_net_u16 (local->port);
2376           mapping->fib_index = local->fib_index;
2377           if (m->affinity)
2378             {
2379               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2380                                                 match.protocol, match.port,
2381                                                 tmp[lo], m->affinity,
2382                                                 m->affinity_per_service_list_head_index))
2383                 nat_log_info ("create affinity record failed");
2384             }
2385           vec_free (tmp);
2386         }
2387       else
2388         {
2389           if (PREDICT_FALSE (lb != 0))
2390             *lb = NO_LB_NAT;
2391           mapping->fib_index = m->fib_index;
2392           mapping->addr = m->local_addr;
2393           /* Address only mapping doesn't change port */
2394           mapping->port = is_addr_only_static_mapping (m) ? match.port
2395             : clib_host_to_net_u16 (m->local_port);
2396         }
2397       mapping->protocol = m->proto;
2398     }
2399   else
2400     {
2401       mapping->addr = m->external_addr;
2402       /* Address only mapping doesn't change port */
2403       mapping->port = is_addr_only_static_mapping (m) ? match.port
2404         : clib_host_to_net_u16 (m->external_port);
2405       mapping->fib_index = sm->outside_fib_index;
2406     }
2407
2408 end:
2409   if (PREDICT_FALSE (is_addr_only != 0))
2410     *is_addr_only = is_addr_only_static_mapping (m);
2411
2412   if (PREDICT_FALSE (twice_nat != 0))
2413     *twice_nat = m->twice_nat;
2414
2415   if (PREDICT_FALSE (is_identity_nat != 0))
2416     *is_identity_nat = is_identity_static_mapping (m);
2417
2418   return 0;
2419 }
2420
2421 static_always_inline u16
2422 snat_random_port (u16 min, u16 max)
2423 {
2424   snat_main_t *sm = &snat_main;
2425   return min + random_u32 (&sm->random_seed) /
2426     (random_u32_max () / (max - min + 1) + 1);
2427 }
2428
2429 int
2430 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2431                                      u32 fib_index,
2432                                      u32 thread_index,
2433                                      snat_session_key_t * k,
2434                                      u16 port_per_thread,
2435                                      u32 snat_thread_index)
2436 {
2437   snat_main_t *sm = &snat_main;
2438
2439   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2440                                   port_per_thread, snat_thread_index);
2441 }
2442
2443 static int
2444 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2445                                  u32 fib_index,
2446                                  u32 thread_index,
2447                                  snat_session_key_t * k,
2448                                  u16 port_per_thread, u32 snat_thread_index)
2449 {
2450   int i;
2451   snat_address_t *a, *ga = 0;
2452   u32 portnum;
2453
2454   for (i = 0; i < vec_len (addresses); i++)
2455     {
2456       a = addresses + i;
2457       switch (k->protocol)
2458         {
2459 #define _(N, j, n, s) \
2460         case SNAT_PROTOCOL_##N: \
2461           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2462             { \
2463               if (a->fib_index == fib_index) \
2464                 { \
2465                   while (1) \
2466                     { \
2467                       portnum = (port_per_thread * \
2468                         snat_thread_index) + \
2469                         snat_random_port(1, port_per_thread) + 1024; \
2470                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2471                         continue; \
2472                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2473                       a->busy_##n##_ports_per_thread[thread_index]++; \
2474                       a->busy_##n##_ports++; \
2475                       k->addr = a->addr; \
2476                       k->port = clib_host_to_net_u16(portnum); \
2477                       return 0; \
2478                     } \
2479                 } \
2480               else if (a->fib_index == ~0) \
2481                 { \
2482                   ga = a; \
2483                 } \
2484             } \
2485           break;
2486           foreach_snat_protocol
2487 #undef _
2488         default:
2489           nat_log_info ("unknown protocol");
2490           return 1;
2491         }
2492
2493     }
2494
2495   if (ga)
2496     {
2497       a = ga;
2498       switch (k->protocol)
2499         {
2500 #define _(N, j, n, s) \
2501         case SNAT_PROTOCOL_##N: \
2502           while (1) \
2503             { \
2504               portnum = (port_per_thread * \
2505                 snat_thread_index) + \
2506                 snat_random_port(1, port_per_thread) + 1024; \
2507               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2508                 continue; \
2509               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2510               a->busy_##n##_ports_per_thread[thread_index]++; \
2511               a->busy_##n##_ports++; \
2512               k->addr = a->addr; \
2513               k->port = clib_host_to_net_u16(portnum); \
2514               return 0; \
2515             }
2516           break;
2517           foreach_snat_protocol
2518 #undef _
2519         default:
2520           nat_log_info ("unknown protocol");
2521           return 1;
2522         }
2523     }
2524
2525   /* Totally out of translations to use... */
2526   snat_ipfix_logging_addresses_exhausted (0);
2527   return 1;
2528 }
2529
2530 static int
2531 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2532                               u32 fib_index,
2533                               u32 thread_index,
2534                               snat_session_key_t * k,
2535                               u16 port_per_thread, u32 snat_thread_index)
2536 {
2537   snat_main_t *sm = &snat_main;
2538   snat_address_t *a = addresses;
2539   u16 m, ports, portnum, A, j;
2540   m = 16 - (sm->psid_offset + sm->psid_length);
2541   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2542
2543   if (!vec_len (addresses))
2544     goto exhausted;
2545
2546   switch (k->protocol)
2547     {
2548 #define _(N, i, n, s) \
2549     case SNAT_PROTOCOL_##N: \
2550       if (a->busy_##n##_ports < ports) \
2551         { \
2552           while (1) \
2553             { \
2554               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2555               j = snat_random_port(0, pow2_mask(m)); \
2556               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2557               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2558                 continue; \
2559               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2560               a->busy_##n##_ports++; \
2561               k->addr = a->addr; \
2562               k->port = clib_host_to_net_u16 (portnum); \
2563               return 0; \
2564             } \
2565         } \
2566       break;
2567       foreach_snat_protocol
2568 #undef _
2569     default:
2570       nat_log_info ("unknown protocol");
2571       return 1;
2572     }
2573
2574 exhausted:
2575   /* Totally out of translations to use... */
2576   snat_ipfix_logging_addresses_exhausted (0);
2577   return 1;
2578 }
2579
2580 static int
2581 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2582                                u32 fib_index,
2583                                u32 thread_index,
2584                                snat_session_key_t * k,
2585                                u16 port_per_thread, u32 snat_thread_index)
2586 {
2587   snat_main_t *sm = &snat_main;
2588   snat_address_t *a = addresses;
2589   u16 portnum, ports;
2590
2591   ports = sm->end_port - sm->start_port + 1;
2592
2593   if (!vec_len (addresses))
2594     goto exhausted;
2595
2596   switch (k->protocol)
2597     {
2598 #define _(N, i, n, s) \
2599     case SNAT_PROTOCOL_##N: \
2600       if (a->busy_##n##_ports < ports) \
2601         { \
2602           while (1) \
2603             { \
2604               portnum = snat_random_port(sm->start_port, sm->end_port); \
2605               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2606                 continue; \
2607               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2608               a->busy_##n##_ports++; \
2609               k->addr = a->addr; \
2610               k->port = clib_host_to_net_u16 (portnum); \
2611               return 0; \
2612             } \
2613         } \
2614       break;
2615       foreach_snat_protocol
2616 #undef _
2617     default:
2618       nat_log_info ("unknown protocol");
2619       return 1;
2620     }
2621
2622 exhausted:
2623   /* Totally out of translations to use... */
2624   snat_ipfix_logging_addresses_exhausted (0);
2625   return 1;
2626 }
2627
2628 void
2629 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2630 {
2631   dpo_id_t dpo_v4 = DPO_INVALID;
2632   fib_prefix_t pfx = {
2633     .fp_proto = FIB_PROTOCOL_IP4,
2634     .fp_len = 32,
2635     .fp_addr.ip4.as_u32 = addr.as_u32,
2636   };
2637
2638   if (is_add)
2639     {
2640       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2641       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2642                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2643       dpo_reset (&dpo_v4);
2644     }
2645   else
2646     {
2647       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2648     }
2649 }
2650
2651 u8 *
2652 format_session_kvp (u8 * s, va_list * args)
2653 {
2654   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2655   snat_session_key_t k;
2656
2657   k.as_u64 = v->key;
2658
2659   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2660
2661   return s;
2662 }
2663
2664 u8 *
2665 format_static_mapping_kvp (u8 * s, va_list * args)
2666 {
2667   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2668   snat_session_key_t k;
2669
2670   k.as_u64 = v->key;
2671
2672   s = format (s, "%U static-mapping-index %llu",
2673               format_static_mapping_key, &k, v->value);
2674
2675   return s;
2676 }
2677
2678 u8 *
2679 format_user_kvp (u8 * s, va_list * args)
2680 {
2681   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2682   snat_user_key_t k;
2683
2684   k.as_u64 = v->key;
2685
2686   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2687               k.fib_index, v->value);
2688
2689   return s;
2690 }
2691
2692 u8 *
2693 format_ed_session_kvp (u8 * s, va_list * args)
2694 {
2695   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2696   nat_ed_ses_key_t k;
2697
2698   k.as_u64[0] = v->key[0];
2699   k.as_u64[1] = v->key[1];
2700
2701   s =
2702     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2703             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2704             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2705             format_ip_protocol, k.proto, k.fib_index, v->value);
2706
2707   return s;
2708 }
2709
2710 static u32
2711 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2712 {
2713   snat_main_t *sm = &snat_main;
2714   u32 next_worker_index = 0;
2715   u32 hash;
2716
2717   next_worker_index = sm->first_worker_index;
2718   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2719     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2720
2721   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2722     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2723   else
2724     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2725
2726   return next_worker_index;
2727 }
2728
2729 static u32
2730 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2731 {
2732   snat_main_t *sm = &snat_main;
2733   udp_header_t *udp;
2734   u16 port;
2735   snat_session_key_t m_key;
2736   clib_bihash_kv_8_8_t kv, value;
2737   snat_static_mapping_t *m;
2738   u32 proto;
2739   u32 next_worker_index = 0;
2740
2741   /* first try static mappings without port */
2742   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2743     {
2744       m_key.addr = ip0->dst_address;
2745       m_key.port = 0;
2746       m_key.protocol = 0;
2747       m_key.fib_index = rx_fib_index0;
2748       kv.key = m_key.as_u64;
2749       if (!clib_bihash_search_8_8
2750           (&sm->static_mapping_by_external, &kv, &value))
2751         {
2752           m = pool_elt_at_index (sm->static_mappings, value.value);
2753           return m->workers[0];
2754         }
2755     }
2756
2757   proto = ip_proto_to_snat_proto (ip0->protocol);
2758   udp = ip4_next_header (ip0);
2759   port = udp->dst_port;
2760
2761   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2762     {
2763       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2764         return vlib_get_thread_index ();
2765
2766       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2767         {
2768           nat_reass_ip4_t *reass;
2769
2770           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2771                                       ip0->fragment_id, ip0->protocol);
2772
2773           if (reass && (reass->thread_index != (u32) ~ 0))
2774             return reass->thread_index;
2775           else
2776             return vlib_get_thread_index ();
2777         }
2778     }
2779
2780   /* unknown protocol */
2781   if (PREDICT_FALSE (proto == ~0))
2782     {
2783       /* use current thread */
2784       return vlib_get_thread_index ();
2785     }
2786
2787   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2788     {
2789       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2790       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2791       if (!icmp_is_error_message (icmp))
2792         port = echo->identifier;
2793       else
2794         {
2795           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2796           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2797           void *l4_header = ip4_next_header (inner_ip);
2798           switch (proto)
2799             {
2800             case SNAT_PROTOCOL_ICMP:
2801               icmp = (icmp46_header_t *) l4_header;
2802               echo = (icmp_echo_header_t *) (icmp + 1);
2803               port = echo->identifier;
2804               break;
2805             case SNAT_PROTOCOL_UDP:
2806             case SNAT_PROTOCOL_TCP:
2807               port = ((tcp_udp_header_t *) l4_header)->src_port;
2808               break;
2809             default:
2810               return vlib_get_thread_index ();
2811             }
2812         }
2813     }
2814
2815   /* try static mappings with port */
2816   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2817     {
2818       m_key.addr = ip0->dst_address;
2819       m_key.port = clib_net_to_host_u16 (port);
2820       m_key.protocol = proto;
2821       m_key.fib_index = rx_fib_index0;
2822       kv.key = m_key.as_u64;
2823       if (!clib_bihash_search_8_8
2824           (&sm->static_mapping_by_external, &kv, &value))
2825         {
2826           m = pool_elt_at_index (sm->static_mappings, value.value);
2827           return m->workers[0];
2828         }
2829     }
2830
2831   /* worker by outside port */
2832   next_worker_index = sm->first_worker_index;
2833   next_worker_index +=
2834     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2835   return next_worker_index;
2836 }
2837
2838 static u32
2839 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2840 {
2841   snat_main_t *sm = &snat_main;
2842   clib_bihash_kv_8_8_t kv, value;
2843   u32 proto, next_worker_index = 0;
2844   udp_header_t *udp;
2845   u16 port;
2846   snat_static_mapping_t *m;
2847   u32 hash;
2848
2849   /* first try static mappings without port */
2850   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2851     {
2852       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2853       if (!clib_bihash_search_8_8
2854           (&sm->static_mapping_by_external, &kv, &value))
2855         {
2856           m = pool_elt_at_index (sm->static_mappings, value.value);
2857           return m->workers[0];
2858         }
2859     }
2860
2861   proto = ip_proto_to_snat_proto (ip->protocol);
2862
2863   /* unknown protocol */
2864   if (PREDICT_FALSE (proto == ~0))
2865     {
2866       /* use current thread */
2867       return vlib_get_thread_index ();
2868     }
2869
2870   udp = ip4_next_header (ip);
2871   port = udp->dst_port;
2872
2873   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2874     {
2875       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2876       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2877       if (!icmp_is_error_message (icmp))
2878         port = echo->identifier;
2879       else
2880         {
2881           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2882           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2883           void *l4_header = ip4_next_header (inner_ip);
2884           switch (proto)
2885             {
2886             case SNAT_PROTOCOL_ICMP:
2887               icmp = (icmp46_header_t *) l4_header;
2888               echo = (icmp_echo_header_t *) (icmp + 1);
2889               port = echo->identifier;
2890               break;
2891             case SNAT_PROTOCOL_UDP:
2892             case SNAT_PROTOCOL_TCP:
2893               port = ((tcp_udp_header_t *) l4_header)->src_port;
2894               break;
2895             default:
2896               return vlib_get_thread_index ();
2897             }
2898         }
2899     }
2900
2901   /* try static mappings with port */
2902   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2903     {
2904       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2905                   clib_net_to_host_u16 (port));
2906       if (!clib_bihash_search_8_8
2907           (&sm->static_mapping_by_external, &kv, &value))
2908         {
2909           m = pool_elt_at_index (sm->static_mappings, value.value);
2910           if (!is_lb_static_mapping (m))
2911             return m->workers[0];
2912
2913           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2914             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
2915
2916           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2917             return m->workers[hash & (_vec_len (m->workers) - 1)];
2918           else
2919             return m->workers[hash % _vec_len (m->workers)];
2920         }
2921     }
2922
2923   /* worker by outside port */
2924   next_worker_index = sm->first_worker_index;
2925   next_worker_index +=
2926     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2927
2928   return next_worker_index;
2929 }
2930
2931 static clib_error_t *
2932 snat_config (vlib_main_t * vm, unformat_input_t * input)
2933 {
2934   snat_main_t *sm = &snat_main;
2935   nat66_main_t *nm = &nat66_main;
2936   u32 translation_buckets = 1024;
2937   u32 translation_memory_size = 128 << 20;
2938   u32 user_buckets = 128;
2939   u32 user_memory_size = 64 << 20;
2940   u32 max_translations_per_user = 100;
2941   u32 outside_vrf_id = 0;
2942   u32 outside_ip6_vrf_id = 0;
2943   u32 inside_vrf_id = 0;
2944   u32 static_mapping_buckets = 1024;
2945   u32 static_mapping_memory_size = 64 << 20;
2946   u32 nat64_bib_buckets = 1024;
2947   u32 nat64_bib_memory_size = 128 << 20;
2948   u32 nat64_st_buckets = 2048;
2949   u32 nat64_st_memory_size = 256 << 20;
2950   u8 static_mapping_only = 0;
2951   u8 static_mapping_connection_tracking = 0;
2952   snat_main_per_thread_data_t *tsm;
2953   dslite_main_t *dm = &dslite_main;
2954
2955   sm->deterministic = 0;
2956   sm->out2in_dpo = 0;
2957   sm->endpoint_dependent = 0;
2958
2959   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2960     {
2961       if (unformat
2962           (input, "translation hash buckets %d", &translation_buckets))
2963         ;
2964       else if (unformat (input, "translation hash memory %d",
2965                          &translation_memory_size));
2966       else if (unformat (input, "user hash buckets %d", &user_buckets))
2967         ;
2968       else if (unformat (input, "user hash memory %d", &user_memory_size))
2969         ;
2970       else if (unformat (input, "max translations per user %d",
2971                          &max_translations_per_user))
2972         ;
2973       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
2974         ;
2975       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
2976         ;
2977       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
2978         ;
2979       else if (unformat (input, "static mapping only"))
2980         {
2981           static_mapping_only = 1;
2982           if (unformat (input, "connection tracking"))
2983             static_mapping_connection_tracking = 1;
2984         }
2985       else if (unformat (input, "deterministic"))
2986         sm->deterministic = 1;
2987       else if (unformat (input, "nat64 bib hash buckets %d",
2988                          &nat64_bib_buckets))
2989         ;
2990       else if (unformat (input, "nat64 bib hash memory %d",
2991                          &nat64_bib_memory_size))
2992         ;
2993       else
2994         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2995         ;
2996       else if (unformat (input, "nat64 st hash memory %d",
2997                          &nat64_st_memory_size))
2998         ;
2999       else if (unformat (input, "out2in dpo"))
3000         sm->out2in_dpo = 1;
3001       else if (unformat (input, "dslite ce"))
3002         dslite_set_ce (dm, 1);
3003       else if (unformat (input, "endpoint-dependent"))
3004         sm->endpoint_dependent = 1;
3005       else
3006         return clib_error_return (0, "unknown input '%U'",
3007                                   format_unformat_error, input);
3008     }
3009
3010   if (sm->deterministic && sm->endpoint_dependent)
3011     return clib_error_return (0,
3012                               "deterministic and endpoint-dependent modes are mutually exclusive");
3013
3014   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3015     return clib_error_return (0,
3016                               "static mapping only mode available only for simple nat");
3017
3018   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3019     return clib_error_return (0,
3020                               "out2in dpo mode available only for simple nat");
3021
3022   /* for show commands, etc. */
3023   sm->translation_buckets = translation_buckets;
3024   sm->translation_memory_size = translation_memory_size;
3025   /* do not exceed load factor 10 */
3026   sm->max_translations = 10 * translation_buckets;
3027   sm->user_buckets = user_buckets;
3028   sm->user_memory_size = user_memory_size;
3029   sm->max_translations_per_user = max_translations_per_user;
3030   sm->outside_vrf_id = outside_vrf_id;
3031   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3032                                                              outside_vrf_id,
3033                                                              FIB_SOURCE_PLUGIN_HI);
3034   nm->outside_vrf_id = outside_ip6_vrf_id;
3035   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3036                                                              outside_ip6_vrf_id,
3037                                                              FIB_SOURCE_PLUGIN_HI);
3038   sm->inside_vrf_id = inside_vrf_id;
3039   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3040                                                             inside_vrf_id,
3041                                                             FIB_SOURCE_PLUGIN_HI);
3042   sm->static_mapping_only = static_mapping_only;
3043   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3044
3045   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3046                   nat64_st_memory_size);
3047
3048   if (sm->deterministic)
3049     {
3050       sm->in2out_node_index = snat_det_in2out_node.index;
3051       sm->in2out_output_node_index = ~0;
3052       sm->out2in_node_index = snat_det_out2in_node.index;
3053       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3054       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3055     }
3056   else
3057     {
3058       if (sm->endpoint_dependent)
3059         {
3060           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3061           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3062           sm->in2out_node_index = nat44_ed_in2out_node.index;
3063           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3064           sm->out2in_node_index = nat44_ed_out2in_node.index;
3065           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3066           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3067           nat_affinity_init (vm);
3068         }
3069       else
3070         {
3071           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3072           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3073           sm->in2out_node_index = snat_in2out_node.index;
3074           sm->in2out_output_node_index = snat_in2out_output_node.index;
3075           sm->out2in_node_index = snat_out2in_node.index;
3076           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3077           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3078         }
3079       if (!static_mapping_only ||
3080           (static_mapping_only && static_mapping_connection_tracking))
3081         {
3082           /* *INDENT-OFF* */
3083           vec_foreach (tsm, sm->per_thread_data)
3084             {
3085               if (sm->endpoint_dependent)
3086                 {
3087                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3088                                          translation_buckets,
3089                                          translation_memory_size);
3090                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3091                                                       format_ed_session_kvp);
3092
3093                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3094                                          translation_buckets,
3095                                          translation_memory_size);
3096                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3097                                                       format_ed_session_kvp);
3098                 }
3099               else
3100                 {
3101                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3102                                         translation_buckets,
3103                                         translation_memory_size);
3104                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3105                                                      format_session_kvp);
3106
3107                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3108                                         translation_buckets,
3109                                         translation_memory_size);
3110                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3111                                                      format_session_kvp);
3112                 }
3113
3114               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3115                                     user_memory_size);
3116               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3117                                                  format_user_kvp);
3118             }
3119           /* *INDENT-ON* */
3120
3121         }
3122       else
3123         {
3124           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3125           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3126         }
3127       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3128                             "static_mapping_by_local", static_mapping_buckets,
3129                             static_mapping_memory_size);
3130       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3131                                          format_static_mapping_kvp);
3132
3133       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3134                             "static_mapping_by_external",
3135                             static_mapping_buckets,
3136                             static_mapping_memory_size);
3137       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3138                                          format_static_mapping_kvp);
3139     }
3140
3141   return 0;
3142 }
3143
3144 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3145
3146 static void
3147 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3148                                  uword opaque,
3149                                  u32 sw_if_index,
3150                                  ip4_address_t * address,
3151                                  u32 address_length,
3152                                  u32 if_address_index, u32 is_delete)
3153 {
3154   snat_main_t *sm = &snat_main;
3155   snat_static_map_resolve_t *rp;
3156   snat_static_mapping_t *m;
3157   snat_session_key_t m_key;
3158   clib_bihash_kv_8_8_t kv, value;
3159   int i, rv;
3160   ip4_address_t l_addr;
3161
3162   for (i = 0; i < vec_len (sm->to_resolve); i++)
3163     {
3164       rp = sm->to_resolve + i;
3165       if (rp->addr_only == 0)
3166         continue;
3167       if (rp->sw_if_index == sw_if_index)
3168         goto match;
3169     }
3170
3171   return;
3172
3173 match:
3174   m_key.addr.as_u32 = address->as_u32;
3175   m_key.port = rp->addr_only ? 0 : rp->e_port;
3176   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3177   m_key.fib_index = sm->outside_fib_index;
3178   kv.key = m_key.as_u64;
3179   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3180     m = 0;
3181   else
3182     m = pool_elt_at_index (sm->static_mappings, value.value);
3183
3184   if (!is_delete)
3185     {
3186       /* Don't trip over lease renewal, static config */
3187       if (m)
3188         return;
3189     }
3190   else
3191     {
3192       if (!m)
3193         return;
3194     }
3195
3196   /* Indetity mapping? */
3197   if (rp->l_addr.as_u32 == 0)
3198     l_addr.as_u32 = address[0].as_u32;
3199   else
3200     l_addr.as_u32 = rp->l_addr.as_u32;
3201   /* Add the static mapping */
3202   rv = snat_add_static_mapping (l_addr,
3203                                 address[0],
3204                                 rp->l_port,
3205                                 rp->e_port,
3206                                 rp->vrf_id,
3207                                 rp->addr_only, ~0 /* sw_if_index */ ,
3208                                 rp->proto, !is_delete, rp->twice_nat,
3209                                 rp->out2in_only, rp->tag, rp->identity_nat);
3210   if (rv)
3211     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3212 }
3213
3214 static void
3215 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3216                                        uword opaque,
3217                                        u32 sw_if_index,
3218                                        ip4_address_t * address,
3219                                        u32 address_length,
3220                                        u32 if_address_index, u32 is_delete)
3221 {
3222   snat_main_t *sm = &snat_main;
3223   snat_static_map_resolve_t *rp;
3224   ip4_address_t l_addr;
3225   int i, j;
3226   int rv;
3227   u8 twice_nat = 0;
3228   snat_address_t *addresses = sm->addresses;
3229
3230   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3231     {
3232       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3233         goto match;
3234     }
3235
3236   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3237     {
3238       twice_nat = 1;
3239       addresses = sm->twice_nat_addresses;
3240       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3241         goto match;
3242     }
3243
3244   return;
3245
3246 match:
3247   if (!is_delete)
3248     {
3249       /* Don't trip over lease renewal, static config */
3250       for (j = 0; j < vec_len (addresses); j++)
3251         if (addresses[j].addr.as_u32 == address->as_u32)
3252           return;
3253
3254       (void) snat_add_address (sm, address, ~0, twice_nat);
3255       /* Scan static map resolution vector */
3256       for (j = 0; j < vec_len (sm->to_resolve); j++)
3257         {
3258           rp = sm->to_resolve + j;
3259           if (rp->addr_only)
3260             continue;
3261           /* On this interface? */
3262           if (rp->sw_if_index == sw_if_index)
3263             {
3264               /* Indetity mapping? */
3265               if (rp->l_addr.as_u32 == 0)
3266                 l_addr.as_u32 = address[0].as_u32;
3267               else
3268                 l_addr.as_u32 = rp->l_addr.as_u32;
3269               /* Add the static mapping */
3270               rv = snat_add_static_mapping (l_addr,
3271                                             address[0],
3272                                             rp->l_port,
3273                                             rp->e_port,
3274                                             rp->vrf_id,
3275                                             rp->addr_only,
3276                                             ~0 /* sw_if_index */ ,
3277                                             rp->proto,
3278                                             rp->is_add, rp->twice_nat,
3279                                             rp->out2in_only, rp->tag,
3280                                             rp->identity_nat);
3281               if (rv)
3282                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3283             }
3284         }
3285       return;
3286     }
3287   else
3288     {
3289       (void) snat_del_address (sm, address[0], 1, twice_nat);
3290       return;
3291     }
3292 }
3293
3294
3295 int
3296 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3297                             u8 twice_nat)
3298 {
3299   ip4_main_t *ip4_main = sm->ip4_main;
3300   ip4_address_t *first_int_addr;
3301   snat_static_map_resolve_t *rp;
3302   u32 *indices_to_delete = 0;
3303   int i, j;
3304   u32 *auto_add_sw_if_indices =
3305     twice_nat ? sm->
3306     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3307
3308   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3309     );
3310
3311   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3312     {
3313       if (auto_add_sw_if_indices[i] == sw_if_index)
3314         {
3315           if (is_del)
3316             {
3317               /* if have address remove it */
3318               if (first_int_addr)
3319                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3320               else
3321                 {
3322                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3323                     {
3324                       rp = sm->to_resolve + j;
3325                       if (rp->sw_if_index == sw_if_index)
3326                         vec_add1 (indices_to_delete, j);
3327                     }
3328                   if (vec_len (indices_to_delete))
3329                     {
3330                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3331                         vec_del1 (sm->to_resolve, j);
3332                       vec_free (indices_to_delete);
3333                     }
3334                 }
3335               if (twice_nat)
3336                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3337               else
3338                 vec_del1 (sm->auto_add_sw_if_indices, i);
3339             }
3340           else
3341             return VNET_API_ERROR_VALUE_EXIST;
3342
3343           return 0;
3344         }
3345     }
3346
3347   if (is_del)
3348     return VNET_API_ERROR_NO_SUCH_ENTRY;
3349
3350   /* add to the auto-address list */
3351   if (twice_nat)
3352     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3353   else
3354     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3355
3356   /* If the address is already bound - or static - add it now */
3357   if (first_int_addr)
3358     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3359
3360   return 0;
3361 }
3362
3363 int
3364 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3365                    snat_protocol_t proto, u32 vrf_id, int is_in)
3366 {
3367   snat_main_per_thread_data_t *tsm;
3368   clib_bihash_kv_8_8_t kv, value;
3369   ip4_header_t ip;
3370   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3371   snat_session_key_t key;
3372   snat_session_t *s;
3373   clib_bihash_8_8_t *t;
3374
3375   if (sm->endpoint_dependent)
3376     return VNET_API_ERROR_UNSUPPORTED;
3377
3378   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3379   if (sm->num_workers > 1)
3380     tsm =
3381       vec_elt_at_index (sm->per_thread_data,
3382                         sm->worker_in2out_cb (&ip, fib_index));
3383   else
3384     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3385
3386   key.addr.as_u32 = addr->as_u32;
3387   key.port = clib_host_to_net_u16 (port);
3388   key.protocol = proto;
3389   key.fib_index = fib_index;
3390   kv.key = key.as_u64;
3391   t = is_in ? &tsm->in2out : &tsm->out2in;
3392   if (!clib_bihash_search_8_8 (t, &kv, &value))
3393     {
3394       if (pool_is_free_index (tsm->sessions, value.value))
3395         return VNET_API_ERROR_UNSPECIFIED;
3396
3397       s = pool_elt_at_index (tsm->sessions, value.value);
3398       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3399       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3400       return 0;
3401     }
3402
3403   return VNET_API_ERROR_NO_SUCH_ENTRY;
3404 }
3405
3406 int
3407 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3408                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3409                       u32 vrf_id, int is_in)
3410 {
3411   ip4_header_t ip;
3412   clib_bihash_16_8_t *t;
3413   nat_ed_ses_key_t key;
3414   clib_bihash_kv_16_8_t kv, value;
3415   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3416   snat_session_t *s;
3417   snat_main_per_thread_data_t *tsm;
3418
3419   if (!sm->endpoint_dependent)
3420     return VNET_API_ERROR_FEATURE_DISABLED;
3421
3422   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3423   if (sm->num_workers > 1)
3424     tsm =
3425       vec_elt_at_index (sm->per_thread_data,
3426                         sm->worker_in2out_cb (&ip, fib_index));
3427   else
3428     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3429
3430   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3431   key.l_addr.as_u32 = addr->as_u32;
3432   key.r_addr.as_u32 = eh_addr->as_u32;
3433   key.l_port = clib_host_to_net_u16 (port);
3434   key.r_port = clib_host_to_net_u16 (eh_port);
3435   key.proto = proto;
3436   key.fib_index = fib_index;
3437   kv.key[0] = key.as_u64[0];
3438   kv.key[1] = key.as_u64[1];
3439   if (clib_bihash_search_16_8 (t, &kv, &value))
3440     return VNET_API_ERROR_NO_SUCH_ENTRY;
3441
3442   if (pool_is_free_index (tsm->sessions, value.value))
3443     return VNET_API_ERROR_UNSPECIFIED;
3444   s = pool_elt_at_index (tsm->sessions, value.value);
3445   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3446   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3447   return 0;
3448 }
3449
3450 void
3451 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3452 {
3453   snat_main_t *sm = &snat_main;
3454
3455   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3456   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3457   sm->psid = psid;
3458   sm->psid_offset = psid_offset;
3459   sm->psid_length = psid_length;
3460 }
3461
3462 void
3463 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3464 {
3465   snat_main_t *sm = &snat_main;
3466
3467   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3468   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3469   sm->start_port = start_port;
3470   sm->end_port = end_port;
3471 }
3472
3473 void
3474 nat_set_alloc_addr_and_port_default (void)
3475 {
3476   snat_main_t *sm = &snat_main;
3477
3478   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3479   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3480 }
3481
3482 /*
3483  * fd.io coding-style-patch-verification: ON
3484  *
3485  * Local Variables:
3486  * eval: (c-set-style "gnu")
3487  * End:
3488  */