src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/lib/ipfix_logging.h>
  25 #include <nat/lib/nat_syslog.h>
  26 #include <nat/nat_inlines.h>
  27 #include <nat/nat44/inlines.h>
  28 #include <nat/nat_affinity.h>
  29 #include <vnet/fib/fib_table.h>
  30 #include <vnet/fib/ip4_fib.h>
  31 #include <vnet/ip/reass/ip4_sv_reass.h>
  32 #include <vppinfra/bihash_16_8.h>
  33 #include <nat/nat44/ed_inlines.h>
  34 #include <vnet/ip/ip_table.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_ha.h>
  37 #include <nat/nat44-ei/nat44_ei.h>
  38
  39 #include <vpp/app/version.h>
  40
  41 snat_main_t snat_main;
  42
  43 /* *INDENT-OFF* */
  44 /* Hook up input features */
  45 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  46   .arc_name = "ip4-unicast",
  47   .node_name = "nat-pre-in2out",
  48   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  49                                "ip4-sv-reassembly-feature"),
  50 };
  51 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  52   .arc_name = "ip4-unicast",
  53   .node_name = "nat-pre-out2in",
  54   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  55                                "ip4-dhcp-client-detect",
  56                                "ip4-sv-reassembly-feature"),
  57 };
  58 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  59   .arc_name = "ip4-unicast",
  60   .node_name = "nat44-in2out-worker-handoff",
  61   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  62 };
  63 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  64   .arc_name = "ip4-unicast",
  65   .node_name = "nat44-out2in-worker-handoff",
  66   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  67                                "ip4-dhcp-client-detect"),
  68 };
  69 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  70   .arc_name = "ip4-unicast",
  71   .node_name = "nat44-in2out",
  72   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  73 };
  74 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  75   .arc_name = "ip4-unicast",
  76   .node_name = "nat44-out2in",
  77   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  78                                "ip4-dhcp-client-detect"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-classify",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat44-ed-in2out",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  89 };
  90 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  91   .arc_name = "ip4-unicast",
  92   .node_name = "nat44-ed-out2in",
  93   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  94                                "ip4-dhcp-client-detect"),
  95 };
  96 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-ed-classify",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 100 };
 101 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 102   .arc_name = "ip4-unicast",
 103   .node_name = "nat44-handoff-classify",
 104   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 105 };
 106 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 107   .arc_name = "ip4-unicast",
 108   .node_name = "nat44-in2out-fast",
 109   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 110 };
 111 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 112   .arc_name = "ip4-unicast",
 113   .node_name = "nat44-out2in-fast",
 114   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 115                                "ip4-dhcp-client-detect"),
 116 };
 117 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 118   .arc_name = "ip4-unicast",
 119   .node_name = "nat44-hairpin-dst",
 120   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 121 };
 122 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 123   .arc_name = "ip4-unicast",
 124   .node_name = "nat44-ed-hairpin-dst",
 125   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 126 };
 127
 128 /* Hook up output features */
 129 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 130   .arc_name = "ip4-output",
 131   .node_name = "nat44-in2out-output",
 132   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 133 };
 134 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 135   .arc_name = "ip4-output",
 136   .node_name = "nat44-in2out-output-worker-handoff",
 137   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 140   .arc_name = "ip4-output",
 141   .node_name = "nat44-hairpin-src",
 142   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 143 };
 144 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 145   .arc_name = "ip4-output",
 146   .node_name = "nat-pre-in2out-output",
 147   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 148   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 149 };
 150 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-ed-in2out-output",
 153   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 154   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 155 };
 156 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 157   .arc_name = "ip4-output",
 158   .node_name = "nat44-ed-hairpin-src",
 159   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 160   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 161 };
 162
 163 /* Hook up ip4-local features */
 164 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 165 {
 166   .arc_name = "ip4-local",
 167   .node_name = "nat44-hairpinning",
 168   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 169 };
 170 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 171 {
 172   .arc_name = "ip4-local",
 173   .node_name = "nat44-ed-hairpinning",
 174   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 175 };
 176
 177
 178 VLIB_PLUGIN_REGISTER () = {
 179     .version = VPP_BUILD_VER,
 180     .description = "Network Address Translation (NAT)",
 181 };
 182 /* *INDENT-ON* */
 183
 184 static u32
 185 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
 186                                u32 rx_fib_index, u8 is_output);
 187
 188 static u32
 189 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
 190                                u8 is_output);
 191
 192 static u32 nat_calc_bihash_buckets (u32 n_elts);
 193
 194 u8 *
 195 format_session_kvp (u8 * s, va_list * args)
 196 {
 197   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 198
 199   s = format (s, "%U session-index %llu", format_snat_key, v->key, v->value);
 200
 201   return s;
 202 }
 203
 204 u8 *
 205 format_static_mapping_kvp (u8 * s, va_list * args)
 206 {
 207   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 208
 209   s = format (s, "%U static-mapping-index %llu",
 210               format_snat_key, v->key, v->value);
 211
 212   return s;
 213 }
 214
 215 u8 *
 216 format_user_kvp (u8 * s, va_list * args)
 217 {
 218   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 219   snat_user_key_t k;
 220
 221   k.as_u64 = v->key;
 222
 223   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
 224               k.fib_index, v->value);
 225
 226   return s;
 227 }
 228
 229 u8 *
 230 format_ed_session_kvp (u8 * s, va_list * args)
 231 {
 232   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 233
 234   u8 proto;
 235   u16 r_port, l_port;
 236   ip4_address_t l_addr, r_addr;
 237   u32 fib_index;
 238
 239   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 240   s =
 241     format (s,
 242             "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
 243             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 244             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 245             format_ip_protocol, proto, fib_index,
 246             ed_value_get_session_index (v), ed_value_get_thread_index (v));
 247
 248   return s;
 249 }
 250
 251 void
 252 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 253                        u8 is_ha)
 254 {
 255   clib_bihash_kv_8_8_t kv;
 256   u8 proto;
 257   u16 r_port, l_port;
 258   ip4_address_t *l_addr, *r_addr;
 259   u32 fib_index = 0;
 260   clib_bihash_kv_16_8_t ed_kv;
 261   snat_main_per_thread_data_t *tsm =
 262     vec_elt_at_index (sm->per_thread_data, thread_index);
 263
 264   if (is_ed_session (s))
 265     {
 266       per_vrf_sessions_unregister_session (s, thread_index);
 267     }
 268
 269   if (is_fwd_bypass_session (s))
 270     {
 271       if (snat_is_unk_proto_session (s))
 272         {
 273           init_ed_k (&ed_kv, s->in2out.addr, 0, s->ext_host_addr, 0, 0,
 274                      s->in2out.port);
 275         }
 276       else
 277         {
 278           l_port = s->in2out.port;
 279           r_port = s->ext_host_port;
 280           l_addr = &s->in2out.addr;
 281           r_addr = &s->ext_host_addr;
 282           proto = nat_proto_to_ip_proto (s->nat_proto);
 283           fib_index = s->in2out.fib_index;
 284           init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index,
 285                      proto);
 286         }
 287       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 288         nat_elog_warn ("in2out_ed key del failed");
 289       return;
 290     }
 291
 292   /* session lookup tables */
 293   if (is_ed_session (s))
 294     {
 295       if (is_affinity_sessions (s))
 296         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 297                              s->nat_proto, s->out2in.port);
 298       l_addr = &s->out2in.addr;
 299       r_addr = &s->ext_host_addr;
 300       fib_index = s->out2in.fib_index;
 301       if (snat_is_unk_proto_session (s))
 302         {
 303           proto = s->in2out.port;
 304           r_port = 0;
 305           l_port = 0;
 306         }
 307       else
 308         {
 309           proto = nat_proto_to_ip_proto (s->nat_proto);
 310           l_port = s->out2in.port;
 311           r_port = s->ext_host_port;
 312         }
 313       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 314       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 315         nat_elog_warn ("out2in_ed key del failed");
 316       l_addr = &s->in2out.addr;
 317       fib_index = s->in2out.fib_index;
 318       if (!snat_is_unk_proto_session (s))
 319         l_port = s->in2out.port;
 320       if (is_twice_nat_session (s))
 321         {
 322           r_addr = &s->ext_host_nat_addr;
 323           r_port = s->ext_host_nat_port;
 324         }
 325       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 326       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 327         nat_elog_warn ("in2out_ed key del failed");
 328
 329       if (!is_ha)
 330         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 331                                &s->in2out.addr, s->in2out.port,
 332                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 333                                &s->out2in.addr, s->out2in.port,
 334                                &s->ext_host_addr, s->ext_host_port,
 335                                s->nat_proto, is_twice_nat_session (s));
 336     }
 337   else
 338     {
 339       init_nat_i2o_k (&kv, s);
 340       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 341         nat_elog_warn ("in2out key del failed");
 342       init_nat_o2i_k (&kv, s);
 343       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 344         nat_elog_warn ("out2in key del failed");
 345
 346       if (!is_ha)
 347         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 348                                  &s->in2out.addr, s->in2out.port,
 349                                  &s->out2in.addr, s->out2in.port,
 350                                  s->nat_proto);
 351     }
 352
 353   if (snat_is_unk_proto_session (s))
 354     return;
 355
 356   if (!is_ha)
 357     {
 358       /* log NAT event */
 359       nat_ipfix_logging_nat44_ses_delete (thread_index,
 360                                           s->in2out.addr.as_u32,
 361                                           s->out2in.addr.as_u32,
 362                                           s->nat_proto,
 363                                           s->in2out.port,
 364                                           s->out2in.port,
 365                                           s->in2out.fib_index);
 366
 367       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 368                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 369                    thread_index);
 370     }
 371
 372   /* Twice NAT address and port for external host */
 373   if (is_twice_nat_session (s))
 374     {
 375       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 376                                           thread_index,
 377                                           &s->ext_host_nat_addr,
 378                                           s->ext_host_nat_port, s->nat_proto);
 379     }
 380
 381   if (snat_is_session_static (s))
 382     return;
 383
 384   snat_free_outside_address_and_port (sm->addresses, thread_index,
 385                                       &s->out2in.addr, s->out2in.port,
 386                                       s->nat_proto);
 387 }
 388
 389 snat_user_t *
 390 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 391                         u32 thread_index)
 392 {
 393   snat_user_t *u = 0;
 394   snat_user_key_t user_key;
 395   clib_bihash_kv_8_8_t kv, value;
 396   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 397   dlist_elt_t *per_user_list_head_elt;
 398
 399   user_key.addr.as_u32 = addr->as_u32;
 400   user_key.fib_index = fib_index;
 401   kv.key = user_key.as_u64;
 402
 403   /* Ever heard of the "user" = src ip4 address before? */
 404   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 405     {
 406       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 407         {
 408           vlib_increment_simple_counter (&sm->user_limit_reached,
 409                                          thread_index, 0, 1);
 410           nat_elog_warn ("maximum user limit reached");
 411           return NULL;
 412         }
 413       /* no, make a new one */
 414       pool_get (tsm->users, u);
 415       clib_memset (u, 0, sizeof (*u));
 416
 417       u->addr.as_u32 = addr->as_u32;
 418       u->fib_index = fib_index;
 419
 420       pool_get (tsm->list_pool, per_user_list_head_elt);
 421
 422       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 423         tsm->list_pool;
 424
 425       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 426
 427       kv.value = u - tsm->users;
 428
 429       /* add user */
 430       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 431         {
 432           nat_elog_warn ("user_hash key add failed");
 433           nat44_delete_user_with_no_session (sm, u, thread_index);
 434           return NULL;
 435         }
 436
 437       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 438                                pool_elts (tsm->users));
 439     }
 440   else
 441     {
 442       u = pool_elt_at_index (tsm->users, value.value);
 443     }
 444
 445   return u;
 446 }
 447
 448 // only NAT EI
 449 snat_session_t *
 450 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 451                               u32 thread_index, f64 now)
 452 {
 453   snat_session_t *s;
 454   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 455   u32 oldest_per_user_translation_list_index, session_index;
 456   dlist_elt_t *oldest_per_user_translation_list_elt;
 457   dlist_elt_t *per_user_translation_list_elt;
 458
 459   /* Over quota? Recycle the least recently used translation */
 460   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 461     {
 462       oldest_per_user_translation_list_index =
 463         clib_dlist_remove_head (tsm->list_pool,
 464                                 u->sessions_per_user_list_head_index);
 465
 466       ASSERT (oldest_per_user_translation_list_index != ~0);
 467
 468       /* Add it back to the end of the LRU list */
 469       clib_dlist_addtail (tsm->list_pool,
 470                           u->sessions_per_user_list_head_index,
 471                           oldest_per_user_translation_list_index);
 472       /* Get the list element */
 473       oldest_per_user_translation_list_elt =
 474         pool_elt_at_index (tsm->list_pool,
 475                            oldest_per_user_translation_list_index);
 476
 477       /* Get the session index from the list element */
 478       session_index = oldest_per_user_translation_list_elt->value;
 479
 480       /* Get the session */
 481       s = pool_elt_at_index (tsm->sessions, session_index);
 482
 483       // TODO: ONLY EI version should be called
 484       nat_free_session_data (sm, s, thread_index, 0);
 485       if (snat_is_session_static (s))
 486         u->nstaticsessions--;
 487       else
 488         u->nsessions--;
 489       s->flags = 0;
 490       s->total_bytes = 0;
 491       s->total_pkts = 0;
 492       s->state = 0;
 493       s->ext_host_addr.as_u32 = 0;
 494       s->ext_host_port = 0;
 495       s->ext_host_nat_addr.as_u32 = 0;
 496       s->ext_host_nat_port = 0;
 497     }
 498   else
 499     {
 500       pool_get (tsm->sessions, s);
 501       clib_memset (s, 0, sizeof (*s));
 502
 503       /* Create list elts */
 504       pool_get (tsm->list_pool, per_user_translation_list_elt);
 505       clib_dlist_init (tsm->list_pool,
 506                        per_user_translation_list_elt - tsm->list_pool);
 507
 508       per_user_translation_list_elt->value = s - tsm->sessions;
 509       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 510       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 511
 512       clib_dlist_addtail (tsm->list_pool,
 513                           s->per_user_list_head_index,
 514                           per_user_translation_list_elt - tsm->list_pool);
 515
 516       s->user_index = u - tsm->users;
 517       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 518                                pool_elts (tsm->sessions));
 519     }
 520
 521   s->ha_last_refreshed = now;
 522
 523   return s;
 524 }
 525
 526 void
 527 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 528                           int is_add)
 529 {
 530   snat_main_t *sm = &snat_main;
 531   fib_prefix_t prefix = {
 532     .fp_len = p_len,
 533     .fp_proto = FIB_PROTOCOL_IP4,
 534     .fp_addr = {
 535                 .ip4.as_u32 = addr->as_u32,
 536                 },
 537   };
 538   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 539
 540   if (is_add)
 541     fib_table_entry_update_one_path (fib_index,
 542                                      &prefix,
 543                                      sm->fib_src_low,
 544                                      (FIB_ENTRY_FLAG_CONNECTED |
 545                                       FIB_ENTRY_FLAG_LOCAL |
 546                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 547                                      DPO_PROTO_IP4,
 548                                      NULL,
 549                                      sw_if_index,
 550                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 551   else
 552     fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 553 }
 554
 555 int
 556 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 557                   u8 twice_nat)
 558 {
 559   snat_address_t *ap;
 560   snat_interface_t *i;
 561   vlib_thread_main_t *tm = vlib_get_thread_main ();
 562
 563   if (twice_nat && !sm->endpoint_dependent)
 564     {
 565       nat_log_err ("unsupported");
 566       return VNET_API_ERROR_UNSUPPORTED;
 567     }
 568
 569   /* Check if address already exists */
 570   /* *INDENT-OFF* */
 571   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 572     {
 573       if (ap->addr.as_u32 == addr->as_u32)
 574         {
 575           nat_log_err ("address exist");
 576           return VNET_API_ERROR_VALUE_EXIST;
 577         }
 578     }
 579   /* *INDENT-ON* */
 580
 581   if (twice_nat)
 582     vec_add2 (sm->twice_nat_addresses, ap, 1);
 583   else
 584     vec_add2 (sm->addresses, ap, 1);
 585
 586   ap->addr = *addr;
 587   if (vrf_id != ~0)
 588     ap->fib_index =
 589       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 590                                          sm->fib_src_low);
 591   else
 592     ap->fib_index = ~0;
 593
 594   /* *INDENT-OFF* */
 595   #define _(N, i, n, s) \
 596     clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 597     ap->busy_##n##_ports = 0; \
 598     ap->busy_##n##_ports_per_thread = 0;\
 599     vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 600     foreach_nat_protocol
 601   #undef _
 602   /* *INDENT-ON* */
 603
 604   if (twice_nat)
 605     return 0;
 606
 607   /* Add external address to FIB */
 608   /* *INDENT-OFF* */
 609   pool_foreach (i, sm->interfaces)
 610    {
 611     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 612       continue;
 613
 614     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 615     break;
 616   }
 617   pool_foreach (i, sm->output_feature_interfaces)
 618    {
 619     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 620       continue;
 621
 622     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 623     break;
 624   }
 625   /* *INDENT-ON* */
 626
 627   return 0;
 628 }
 629
 630 static int
 631 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 632 {
 633   snat_static_mapping_t *m;
 634   /* *INDENT-OFF* */
 635   pool_foreach (m, sm->static_mappings)
 636    {
 637       if (is_addr_only_static_mapping (m) ||
 638           is_out2in_only_static_mapping (m) ||
 639           is_identity_static_mapping (m))
 640         continue;
 641       if (m->external_addr.as_u32 == addr.as_u32)
 642         return 1;
 643   }
 644   /* *INDENT-ON* */
 645
 646   return 0;
 647 }
 648
 649 static void
 650 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 651                                        ip4_address_t l_addr,
 652                                        u16 l_port,
 653                                        u32 sw_if_index,
 654                                        u16 e_port,
 655                                        u32 vrf_id,
 656                                        nat_protocol_t proto,
 657                                        int addr_only, int is_add, u8 * tag,
 658                                        int twice_nat, int out2in_only,
 659                                        int identity_nat,
 660                                        ip4_address_t pool_addr, int exact)
 661 {
 662   snat_static_map_resolve_t *rp;
 663
 664   vec_add2 (sm->to_resolve, rp, 1);
 665   rp->l_addr.as_u32 = l_addr.as_u32;
 666   rp->l_port = l_port;
 667   rp->sw_if_index = sw_if_index;
 668   rp->e_port = e_port;
 669   rp->vrf_id = vrf_id;
 670   rp->proto = proto;
 671   rp->addr_only = addr_only;
 672   rp->is_add = is_add;
 673   rp->twice_nat = twice_nat;
 674   rp->out2in_only = out2in_only;
 675   rp->identity_nat = identity_nat;
 676   rp->tag = vec_dup (tag);
 677   rp->pool_addr = pool_addr;
 678   rp->exact = exact;
 679 }
 680
 681 static u32
 682 get_thread_idx_by_port (u16 e_port)
 683 {
 684   snat_main_t *sm = &snat_main;
 685   u32 thread_idx = sm->num_workers;
 686   if (sm->num_workers > 1)
 687     {
 688       thread_idx =
 689         sm->first_worker_index +
 690         sm->workers[(e_port - 1024) / sm->port_per_thread];
 691     }
 692   return thread_idx;
 693 }
 694
 695 void
 696 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 697                                     snat_main_per_thread_data_t * tsm,
 698                                     ip4_address_t l_addr,
 699                                     u16 l_port,
 700                                     u8 protocol,
 701                                     u32 fib_index, int addr_only,
 702                                     ip4_address_t e_addr, u16 e_port)
 703 {
 704   snat_session_t *s;
 705   u32 *indexes_to_free = NULL;
 706   /* *INDENT-OFF* */
 707   pool_foreach (s, tsm->sessions) {
 708     if (s->in2out.fib_index != fib_index ||
 709         s->in2out.addr.as_u32 != l_addr.as_u32)
 710       {
 711         continue;
 712       }
 713     if (!addr_only)
 714       {
 715         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 716             s->out2in.port != e_port ||
 717             s->in2out.port != l_port ||
 718             s->nat_proto != protocol)
 719           continue;
 720       }
 721
 722     if (is_lb_session (s))
 723       continue;
 724     if (!snat_is_session_static (s))
 725       continue;
 726     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 727     vec_add1 (indexes_to_free, s - tsm->sessions);
 728     if (!addr_only)
 729       break;
 730   }
 731   /* *INDENT-ON* */
 732   u32 *ses_index;
 733   vec_foreach (ses_index, indexes_to_free)
 734   {
 735     s = pool_elt_at_index (tsm->sessions, *ses_index);
 736     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 737   }
 738   vec_free (indexes_to_free);
 739 }
 740
 741 int
 742 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 743                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 744                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 745                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 746                          u8 identity_nat, ip4_address_t pool_addr, int exact)
 747 {
 748   snat_main_t *sm = &snat_main;
 749   snat_static_mapping_t *m;
 750   clib_bihash_kv_8_8_t kv, value;
 751   snat_address_t *a = 0;
 752   u32 fib_index = ~0;
 753   snat_interface_t *interface;
 754   int i;
 755   snat_main_per_thread_data_t *tsm;
 756   snat_user_key_t u_key;
 757   snat_user_t *u;
 758   dlist_elt_t *head, *elt;
 759   u32 elt_index, head_index;
 760   u32 ses_index;
 761   u64 user_index;
 762   snat_session_t *s;
 763   snat_static_map_resolve_t *rp, *rp_match = 0;
 764   nat44_lb_addr_port_t *local;
 765   u32 find = ~0;
 766
 767   if (!sm->endpoint_dependent)
 768     {
 769       if (twice_nat || out2in_only)
 770         return VNET_API_ERROR_UNSUPPORTED;
 771     }
 772
 773   /* If the external address is a specific interface address */
 774   if (sw_if_index != ~0)
 775     {
 776       ip4_address_t *first_int_addr;
 777
 778       for (i = 0; i < vec_len (sm->to_resolve); i++)
 779         {
 780           rp = sm->to_resolve + i;
 781           if (rp->sw_if_index != sw_if_index ||
 782               rp->l_addr.as_u32 != l_addr.as_u32 ||
 783               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 784             continue;
 785
 786           if (!addr_only)
 787             {
 788               if ((rp->l_port != l_port && rp->e_port != e_port)
 789                   || rp->proto != proto)
 790                 continue;
 791             }
 792
 793           rp_match = rp;
 794           break;
 795         }
 796
 797       /* Might be already set... */
 798       first_int_addr = ip4_interface_first_address
 799         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 800
 801       if (is_add)
 802         {
 803           if (rp_match)
 804             return VNET_API_ERROR_VALUE_EXIST;
 805
 806           snat_add_static_mapping_when_resolved
 807             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 808              addr_only, is_add, tag, twice_nat, out2in_only,
 809              identity_nat, pool_addr, exact);
 810
 811           /* DHCP resolution required? */
 812           if (first_int_addr == 0)
 813             {
 814               return 0;
 815             }
 816           else
 817             {
 818               e_addr.as_u32 = first_int_addr->as_u32;
 819               /* Identity mapping? */
 820               if (l_addr.as_u32 == 0)
 821                 l_addr.as_u32 = e_addr.as_u32;
 822             }
 823         }
 824       else
 825         {
 826           if (!rp_match)
 827             return VNET_API_ERROR_NO_SUCH_ENTRY;
 828
 829           vec_del1 (sm->to_resolve, i);
 830
 831           if (first_int_addr)
 832             {
 833               e_addr.as_u32 = first_int_addr->as_u32;
 834               /* Identity mapping? */
 835               if (l_addr.as_u32 == 0)
 836                 l_addr.as_u32 = e_addr.as_u32;
 837             }
 838           else
 839             return 0;
 840         }
 841     }
 842
 843   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 844   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 845     m = 0;
 846   else
 847     m = pool_elt_at_index (sm->static_mappings, value.value);
 848
 849   if (is_add)
 850     {
 851       if (m)
 852         {
 853           if (is_identity_static_mapping (m))
 854             {
 855               /* *INDENT-OFF* */
 856               pool_foreach (local, m->locals)
 857                {
 858                 if (local->vrf_id == vrf_id)
 859                   return VNET_API_ERROR_VALUE_EXIST;
 860               }
 861               /* *INDENT-ON* */
 862               pool_get (m->locals, local);
 863               local->vrf_id = vrf_id;
 864               local->fib_index =
 865                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 866                                                    sm->fib_src_low);
 867               init_nat_kv (&kv, m->local_addr, m->local_port,
 868                            local->fib_index, m->proto,
 869                            m - sm->static_mappings);
 870               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 871               return 0;
 872             }
 873           else
 874             return VNET_API_ERROR_VALUE_EXIST;
 875         }
 876
 877       if (twice_nat && addr_only)
 878         return VNET_API_ERROR_UNSUPPORTED;
 879
 880       /* Convert VRF id to FIB index */
 881       if (vrf_id != ~0)
 882         fib_index =
 883           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 884                                              sm->fib_src_low);
 885       /* If not specified use inside VRF id from SNAT plugin startup config */
 886       else
 887         {
 888           fib_index = sm->inside_fib_index;
 889           vrf_id = sm->inside_vrf_id;
 890           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 891         }
 892
 893       if (!(out2in_only || identity_nat))
 894         {
 895           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
 896                       addr_only ? 0 : proto);
 897           if (!clib_bihash_search_8_8
 898               (&sm->static_mapping_by_local, &kv, &value))
 899             return VNET_API_ERROR_VALUE_EXIST;
 900         }
 901
 902       /* Find external address in allocated addresses and reserve port for
 903          address and port pair mapping when dynamic translations enabled */
 904       if (!(addr_only || sm->static_mapping_only || out2in_only))
 905         {
 906           for (i = 0; i < vec_len (sm->addresses); i++)
 907             {
 908               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 909                 {
 910                   a = sm->addresses + i;
 911                   /* External port must be unused */
 912                   switch (proto)
 913                     {
 914 #define _(N, j, n, s) \
 915                     case NAT_PROTOCOL_##N: \
 916                       if (a->busy_##n##_port_refcounts[e_port]) \
 917                         return VNET_API_ERROR_INVALID_VALUE; \
 918                       ++a->busy_##n##_port_refcounts[e_port]; \
 919                       if (e_port > 1024) \
 920                         { \
 921                           a->busy_##n##_ports++; \
 922                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 923                         } \
 924                       break;
 925                       foreach_nat_protocol
 926 #undef _
 927                     default:
 928                       nat_elog_info ("unknown protocol");
 929                       return VNET_API_ERROR_INVALID_VALUE_2;
 930                     }
 931                   break;
 932                 }
 933             }
 934           /* External address must be allocated */
 935           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 936             {
 937               if (sw_if_index != ~0)
 938                 {
 939                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 940                     {
 941                       rp = sm->to_resolve + i;
 942                       if (rp->addr_only)
 943                         continue;
 944                       if (rp->sw_if_index != sw_if_index &&
 945                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 946                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 947                           rp->e_port != e_port && rp->proto != proto)
 948                         continue;
 949
 950                       vec_del1 (sm->to_resolve, i);
 951                       break;
 952                     }
 953                 }
 954               return VNET_API_ERROR_NO_SUCH_ENTRY;
 955             }
 956         }
 957
 958       pool_get (sm->static_mappings, m);
 959       clib_memset (m, 0, sizeof (*m));
 960       m->tag = vec_dup (tag);
 961       m->local_addr = l_addr;
 962       m->external_addr = e_addr;
 963       m->twice_nat = twice_nat;
 964
 965       if (twice_nat == TWICE_NAT && exact)
 966         {
 967           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
 968           m->pool_addr = pool_addr;
 969         }
 970
 971       if (out2in_only)
 972         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 973       if (addr_only)
 974         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 975       if (identity_nat)
 976         {
 977           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 978           pool_get (m->locals, local);
 979           local->vrf_id = vrf_id;
 980           local->fib_index = fib_index;
 981         }
 982       else
 983         {
 984           m->vrf_id = vrf_id;
 985           m->fib_index = fib_index;
 986         }
 987       if (!addr_only)
 988         {
 989           m->local_port = l_port;
 990           m->external_port = e_port;
 991           m->proto = proto;
 992         }
 993
 994       if (sm->num_workers > 1)
 995         {
 996           ip4_header_t ip = {
 997             .src_address = m->local_addr,
 998           };
 999           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1000           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1001         }
1002       else
1003         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1004
1005       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
1006                    m - sm->static_mappings);
1007       if (!out2in_only)
1008         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1009
1010       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1011                    m - sm->static_mappings);
1012       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1013
1014       /* Delete dynamic sessions matching local address (+ local port) */
1015       // TODO: based on type of NAT EI/ED
1016       if (!(sm->static_mapping_only))
1017         {
1018           u_key.addr = m->local_addr;
1019           u_key.fib_index = m->fib_index;
1020           kv.key = u_key.as_u64;
1021           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1022             {
1023               user_index = value.value;
1024               u = pool_elt_at_index (tsm->users, user_index);
1025               if (u->nsessions)
1026                 {
1027                   head_index = u->sessions_per_user_list_head_index;
1028                   head = pool_elt_at_index (tsm->list_pool, head_index);
1029                   elt_index = head->next;
1030                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1031                   ses_index = elt->value;
1032                   while (ses_index != ~0)
1033                     {
1034                       s = pool_elt_at_index (tsm->sessions, ses_index);
1035                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1036                       ses_index = elt->value;
1037
1038                       if (snat_is_session_static (s))
1039                         continue;
1040
1041                       if (!addr_only && s->in2out.port != m->local_port)
1042                         continue;
1043
1044                       nat_free_session_data (sm, s,
1045                                              tsm - sm->per_thread_data, 0);
1046                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1047
1048                       if (!addr_only && !sm->endpoint_dependent)
1049                         break;
1050                     }
1051                 }
1052             }
1053         }
1054     }
1055   else
1056     {
1057       if (!m)
1058         {
1059           if (sw_if_index != ~0)
1060             return 0;
1061           else
1062             return VNET_API_ERROR_NO_SUCH_ENTRY;
1063         }
1064
1065       if (identity_nat)
1066         {
1067           if (vrf_id == ~0)
1068             vrf_id = sm->inside_vrf_id;
1069
1070           /* *INDENT-OFF* */
1071           pool_foreach (local, m->locals)
1072            {
1073             if (local->vrf_id == vrf_id)
1074               find = local - m->locals;
1075           }
1076           /* *INDENT-ON* */
1077           if (find == ~0)
1078             return VNET_API_ERROR_NO_SUCH_ENTRY;
1079
1080           local = pool_elt_at_index (m->locals, find);
1081           fib_index = local->fib_index;
1082           pool_put (m->locals, local);
1083         }
1084       else
1085         fib_index = m->fib_index;
1086
1087       /* Free external address port */
1088       if (!(addr_only || sm->static_mapping_only || out2in_only))
1089         {
1090           for (i = 0; i < vec_len (sm->addresses); i++)
1091             {
1092               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1093                 {
1094                   a = sm->addresses + i;
1095                   switch (proto)
1096                     {
1097 #define _(N, j, n, s) \
1098                     case NAT_PROTOCOL_##N: \
1099                       --a->busy_##n##_port_refcounts[e_port]; \
1100                       if (e_port > 1024) \
1101                         { \
1102                           a->busy_##n##_ports--; \
1103                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1104                         } \
1105                       break;
1106                       foreach_nat_protocol
1107 #undef _
1108                     default:
1109                       nat_elog_info ("unknown protocol");
1110                       return VNET_API_ERROR_INVALID_VALUE_2;
1111                     }
1112                   break;
1113                 }
1114             }
1115         }
1116
1117       if (sm->num_workers > 1)
1118         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1119       else
1120         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1121
1122       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1123       if (!out2in_only)
1124         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1125
1126       /* Delete session(s) for static mapping if exist */
1127       if (!(sm->static_mapping_only) ||
1128           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1129         {
1130           if (sm->endpoint_dependent)
1131             {
1132               nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1133                                                   m->local_port, m->proto,
1134                                                   fib_index, addr_only,
1135                                                   e_addr, e_port);
1136             }
1137           else
1138             {
1139               u_key.addr = m->local_addr;
1140               u_key.fib_index = fib_index;
1141               kv.key = u_key.as_u64;
1142               nat44_ei_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1143                                                     e_addr, e_port);
1144             }
1145         }
1146
1147       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1148       if (pool_elts (m->locals))
1149         return 0;
1150
1151       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1152       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1153
1154       vec_free (m->tag);
1155       vec_free (m->workers);
1156       /* Delete static mapping from pool */
1157       pool_put (sm->static_mappings, m);
1158     }
1159
1160   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1161     return 0;
1162
1163   /* Add/delete external address to FIB */
1164   /* *INDENT-OFF* */
1165   pool_foreach (interface, sm->interfaces)
1166    {
1167     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1168       continue;
1169
1170     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1171     break;
1172   }
1173   pool_foreach (interface, sm->output_feature_interfaces)
1174    {
1175     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1176       continue;
1177
1178     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1179     break;
1180   }
1181   /* *INDENT-ON* */
1182
1183   return 0;
1184 }
1185
1186 int
1187 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1188                                  nat_protocol_t proto,
1189                                  nat44_lb_addr_port_t * locals, u8 is_add,
1190                                  twice_nat_type_t twice_nat, u8 out2in_only,
1191                                  u8 * tag, u32 affinity)
1192 {
1193   snat_main_t *sm = &snat_main;
1194   snat_static_mapping_t *m;
1195   clib_bihash_kv_8_8_t kv, value;
1196   snat_address_t *a = 0;
1197   int i;
1198   nat44_lb_addr_port_t *local;
1199   snat_main_per_thread_data_t *tsm;
1200   snat_session_t *s;
1201   uword *bitmap = 0;
1202
1203   if (!sm->endpoint_dependent)
1204     return VNET_API_ERROR_UNSUPPORTED;
1205
1206   init_nat_k (&kv, e_addr, e_port, 0, proto);
1207   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1208     m = 0;
1209   else
1210     m = pool_elt_at_index (sm->static_mappings, value.value);
1211
1212   if (is_add)
1213     {
1214       if (m)
1215         return VNET_API_ERROR_VALUE_EXIST;
1216
1217       if (vec_len (locals) < 2)
1218         return VNET_API_ERROR_INVALID_VALUE;
1219
1220       /* Find external address in allocated addresses and reserve port for
1221          address and port pair mapping when dynamic translations enabled */
1222       if (!(sm->static_mapping_only || out2in_only))
1223         {
1224           for (i = 0; i < vec_len (sm->addresses); i++)
1225             {
1226               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1227                 {
1228                   a = sm->addresses + i;
1229                   /* External port must be unused */
1230                   switch (proto)
1231                     {
1232 #define _(N, j, n, s) \
1233                     case NAT_PROTOCOL_##N: \
1234                       if (a->busy_##n##_port_refcounts[e_port]) \
1235                         return VNET_API_ERROR_INVALID_VALUE; \
1236                       ++a->busy_##n##_port_refcounts[e_port]; \
1237                       if (e_port > 1024) \
1238                         { \
1239                           a->busy_##n##_ports++; \
1240                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1241                         } \
1242                       break;
1243                       foreach_nat_protocol
1244 #undef _
1245                     default:
1246                       nat_elog_info ("unknown protocol");
1247                       return VNET_API_ERROR_INVALID_VALUE_2;
1248                     }
1249                   break;
1250                 }
1251             }
1252           /* External address must be allocated */
1253           if (!a)
1254             return VNET_API_ERROR_NO_SUCH_ENTRY;
1255         }
1256
1257       pool_get (sm->static_mappings, m);
1258       clib_memset (m, 0, sizeof (*m));
1259       m->tag = vec_dup (tag);
1260       m->external_addr = e_addr;
1261       m->external_port = e_port;
1262       m->proto = proto;
1263       m->twice_nat = twice_nat;
1264       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1265       if (out2in_only)
1266         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1267       m->affinity = affinity;
1268
1269       if (affinity)
1270         m->affinity_per_service_list_head_index =
1271           nat_affinity_get_per_service_list_head_index ();
1272       else
1273         m->affinity_per_service_list_head_index = ~0;
1274
1275       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1276                    m - sm->static_mappings);
1277       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1278         {
1279           nat_elog_err ("static_mapping_by_external key add failed");
1280           return VNET_API_ERROR_UNSPECIFIED;
1281         }
1282
1283       for (i = 0; i < vec_len (locals); i++)
1284         {
1285           locals[i].fib_index =
1286             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1287                                                locals[i].vrf_id,
1288                                                sm->fib_src_low);
1289           if (!out2in_only)
1290             {
1291               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1292                            locals[i].fib_index, m->proto,
1293                            m - sm->static_mappings);
1294               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1295             }
1296           locals[i].prefix = (i == 0) ? locals[i].probability :
1297             (locals[i - 1].prefix + locals[i].probability);
1298           pool_get (m->locals, local);
1299           *local = locals[i];
1300           if (sm->num_workers > 1)
1301             {
1302               ip4_header_t ip = {
1303                 .src_address = locals[i].addr,
1304               };
1305               bitmap =
1306                 clib_bitmap_set (bitmap,
1307                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1308                                  1);
1309             }
1310         }
1311
1312       /* Assign workers */
1313       if (sm->num_workers > 1)
1314         {
1315           /* *INDENT-OFF* */
1316           clib_bitmap_foreach (i, bitmap)
1317              {
1318                vec_add1(m->workers, i);
1319             }
1320           /* *INDENT-ON* */
1321         }
1322     }
1323   else
1324     {
1325       if (!m)
1326         return VNET_API_ERROR_NO_SUCH_ENTRY;
1327
1328       if (!is_lb_static_mapping (m))
1329         return VNET_API_ERROR_INVALID_VALUE;
1330
1331       /* Free external address port */
1332       if (!(sm->static_mapping_only || out2in_only))
1333         {
1334           for (i = 0; i < vec_len (sm->addresses); i++)
1335             {
1336               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1337                 {
1338                   a = sm->addresses + i;
1339                   switch (proto)
1340                     {
1341 #define _(N, j, n, s) \
1342                     case NAT_PROTOCOL_##N: \
1343                       --a->busy_##n##_port_refcounts[e_port]; \
1344                       if (e_port > 1024) \
1345                         { \
1346                           a->busy_##n##_ports--; \
1347                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1348                         } \
1349                       break;
1350                       foreach_nat_protocol
1351 #undef _
1352                     default:
1353                       nat_elog_info ("unknown protocol");
1354                       return VNET_API_ERROR_INVALID_VALUE_2;
1355                     }
1356                   break;
1357                 }
1358             }
1359         }
1360
1361       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1362       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1363         {
1364           nat_elog_err ("static_mapping_by_external key del failed");
1365           return VNET_API_ERROR_UNSPECIFIED;
1366         }
1367
1368       /* *INDENT-OFF* */
1369       pool_foreach (local, m->locals)
1370       {
1371           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1372                             sm->fib_src_low);
1373           if (!out2in_only)
1374             {
1375 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1376               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1377                 {
1378                   nat_elog_err ("static_mapping_by_local key del failed");
1379                   return VNET_API_ERROR_UNSPECIFIED;
1380                 }
1381             }
1382
1383           if (sm->num_workers > 1)
1384             {
1385               ip4_header_t ip = {
1386                 .src_address = local->addr,
1387               };
1388               tsm = vec_elt_at_index (sm->per_thread_data,
1389                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1390             }
1391           else
1392             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1393
1394           /* Delete sessions */
1395           pool_foreach (s, tsm->sessions) {
1396             if (!(is_lb_session (s)))
1397               continue;
1398
1399             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1400                 s->in2out.port != local->port)
1401               continue;
1402
1403             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1404             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1405           }
1406       }
1407       /* *INDENT-ON* */
1408       if (m->affinity)
1409         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1410       pool_free (m->locals);
1411       vec_free (m->tag);
1412       vec_free (m->workers);
1413
1414       pool_put (sm->static_mappings, m);
1415     }
1416
1417   return 0;
1418 }
1419
1420 int
1421 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1422                                        ip4_address_t l_addr, u16 l_port,
1423                                        nat_protocol_t proto, u32 vrf_id,
1424                                        u8 probability, u8 is_add)
1425 {
1426   snat_main_t *sm = &snat_main;
1427   snat_static_mapping_t *m = 0;
1428   clib_bihash_kv_8_8_t kv, value;
1429   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1430   snat_main_per_thread_data_t *tsm;
1431   snat_session_t *s;
1432   u32 *locals = 0;
1433   uword *bitmap = 0;
1434   int i;
1435
1436   if (!sm->endpoint_dependent)
1437     return VNET_API_ERROR_FEATURE_DISABLED;
1438
1439   init_nat_k (&kv, e_addr, e_port, 0, proto);
1440   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1441     m = pool_elt_at_index (sm->static_mappings, value.value);
1442
1443   if (!m)
1444     return VNET_API_ERROR_NO_SUCH_ENTRY;
1445
1446   if (!is_lb_static_mapping (m))
1447     return VNET_API_ERROR_INVALID_VALUE;
1448
1449   /* *INDENT-OFF* */
1450   pool_foreach (local, m->locals)
1451    {
1452     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1453         (local->vrf_id == vrf_id))
1454       {
1455         match_local = local;
1456         break;
1457       }
1458   }
1459   /* *INDENT-ON* */
1460
1461   if (is_add)
1462     {
1463       if (match_local)
1464         return VNET_API_ERROR_VALUE_EXIST;
1465
1466       pool_get (m->locals, local);
1467       clib_memset (local, 0, sizeof (*local));
1468       local->addr.as_u32 = l_addr.as_u32;
1469       local->port = l_port;
1470       local->probability = probability;
1471       local->vrf_id = vrf_id;
1472       local->fib_index =
1473         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1474                                            sm->fib_src_low);
1475
1476       if (!is_out2in_only_static_mapping (m))
1477         {
1478           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto,
1479                        m - sm->static_mappings);
1480           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1481             nat_elog_err ("static_mapping_by_local key add failed");
1482         }
1483     }
1484   else
1485     {
1486       if (!match_local)
1487         return VNET_API_ERROR_NO_SUCH_ENTRY;
1488
1489       if (pool_elts (m->locals) < 3)
1490         return VNET_API_ERROR_UNSPECIFIED;
1491
1492       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1493                         sm->fib_src_low);
1494
1495       if (!is_out2in_only_static_mapping (m))
1496         {
1497           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1498           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1499             nat_elog_err ("static_mapping_by_local key del failed");
1500         }
1501
1502       if (sm->num_workers > 1)
1503         {
1504           ip4_header_t ip = {
1505             .src_address = local->addr,
1506           };
1507           tsm = vec_elt_at_index (sm->per_thread_data,
1508                                   sm->worker_in2out_cb (&ip, m->fib_index,
1509                                                         0));
1510         }
1511       else
1512         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1513
1514       /* Delete sessions */
1515       /* *INDENT-OFF* */
1516       pool_foreach (s, tsm->sessions) {
1517         if (!(is_lb_session (s)))
1518           continue;
1519
1520         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1521             s->in2out.port != match_local->port)
1522           continue;
1523
1524         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1525         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1526       }
1527       /* *INDENT-ON* */
1528
1529       pool_put (m->locals, match_local);
1530     }
1531
1532   vec_free (m->workers);
1533
1534   /* *INDENT-OFF* */
1535   pool_foreach (local, m->locals)
1536    {
1537     vec_add1 (locals, local - m->locals);
1538     if (sm->num_workers > 1)
1539       {
1540         ip4_header_t ip;
1541         ip.src_address.as_u32 = local->addr.as_u32,
1542         bitmap = clib_bitmap_set (bitmap,
1543                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1544                                   1);
1545       }
1546   }
1547   /* *INDENT-ON* */
1548
1549   ASSERT (vec_len (locals) > 1);
1550
1551   local = pool_elt_at_index (m->locals, locals[0]);
1552   local->prefix = local->probability;
1553   for (i = 1; i < vec_len (locals); i++)
1554     {
1555       local = pool_elt_at_index (m->locals, locals[i]);
1556       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1557       local->prefix = local->probability + prev_local->prefix;
1558     }
1559
1560   /* Assign workers */
1561   if (sm->num_workers > 1)
1562     {
1563       /* *INDENT-OFF* */
1564       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1565       /* *INDENT-ON* */
1566     }
1567
1568   return 0;
1569 }
1570
1571 int
1572 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1573                   u8 twice_nat)
1574 {
1575   snat_address_t *a = 0;
1576   snat_session_t *ses;
1577   u32 *ses_to_be_removed = 0, *ses_index;
1578   snat_main_per_thread_data_t *tsm;
1579   snat_static_mapping_t *m;
1580   snat_interface_t *interface;
1581   int i;
1582   snat_address_t *addresses =
1583     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1584
1585   /* Find SNAT address */
1586   for (i = 0; i < vec_len (addresses); i++)
1587     {
1588       if (addresses[i].addr.as_u32 == addr.as_u32)
1589         {
1590           a = addresses + i;
1591           break;
1592         }
1593     }
1594   if (!a)
1595     {
1596       nat_log_err ("no such address");
1597       return VNET_API_ERROR_NO_SUCH_ENTRY;
1598     }
1599
1600   if (delete_sm)
1601     {
1602       ip4_address_t pool_addr = { 0 };
1603       /* *INDENT-OFF* */
1604       pool_foreach (m, sm->static_mappings)
1605        {
1606           if (m->external_addr.as_u32 == addr.as_u32)
1607             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1608                                             m->local_port, m->external_port,
1609                                             m->vrf_id,
1610                                             is_addr_only_static_mapping(m), ~0,
1611                                             m->proto, 0 /* is_add */,
1612                                             m->twice_nat,
1613                                             is_out2in_only_static_mapping(m),
1614                                             m->tag,
1615                                             is_identity_static_mapping(m),
1616                                             pool_addr, 0);
1617       }
1618       /* *INDENT-ON* */
1619     }
1620   else
1621     {
1622       /* Check if address is used in some static mapping */
1623       if (is_snat_address_used_in_static_mapping (sm, addr))
1624         {
1625           nat_log_err ("address used in static mapping");
1626           return VNET_API_ERROR_UNSPECIFIED;
1627         }
1628     }
1629
1630   if (a->fib_index != ~0)
1631     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1632
1633   /* Delete sessions using address */
1634   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1635     {
1636       vec_foreach (tsm, sm->per_thread_data)
1637       {
1638         /* *INDENT-OFF* */
1639         pool_foreach (ses, tsm->sessions)  {
1640           if (ses->out2in.addr.as_u32 == addr.as_u32)
1641             {
1642               nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1643               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1644             }
1645         }
1646         /* *INDENT-ON* */
1647
1648         if (sm->endpoint_dependent)
1649           {
1650             vec_foreach (ses_index, ses_to_be_removed)
1651             {
1652               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1653               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1654             }
1655           }
1656         else
1657           {
1658             vec_foreach (ses_index, ses_to_be_removed)
1659             {
1660               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1661               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1662             }
1663           }
1664
1665         vec_free (ses_to_be_removed);
1666       }
1667     }
1668
1669 #define _(N, i, n, s) \
1670   vec_free (a->busy_##n##_ports_per_thread);
1671   foreach_nat_protocol
1672 #undef _
1673     if (twice_nat)
1674     {
1675       vec_del1 (sm->twice_nat_addresses, i);
1676       return 0;
1677     }
1678   else
1679     vec_del1 (sm->addresses, i);
1680
1681   /* Delete external address from FIB */
1682   /* *INDENT-OFF* */
1683   pool_foreach (interface, sm->interfaces)
1684    {
1685     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1686       continue;
1687
1688     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1689     break;
1690   }
1691   pool_foreach (interface, sm->output_feature_interfaces)
1692    {
1693     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1694       continue;
1695
1696     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1697     break;
1698   }
1699   /* *INDENT-ON* */
1700
1701   return 0;
1702 }
1703
1704 static void
1705 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1706 {
1707 #define _(x)                                                                  \
1708   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1709                                 sw_if_index);                                 \
1710   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1711   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1712                                 sw_if_index);                                 \
1713   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1714   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1715                                 sw_if_index);                                 \
1716   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1717   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1718                                 sw_if_index);                                 \
1719   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1720   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1721                                 sw_if_index);                                 \
1722   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1723   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1724                                 sw_if_index);                                 \
1725   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1726   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1727                                 sw_if_index);                                 \
1728   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1729   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1730                                 sw_if_index);                                 \
1731   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1732   foreach_nat_counter;
1733 #undef _
1734   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1735   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1736 }
1737
1738 void
1739 expire_per_vrf_sessions (u32 fib_index)
1740 {
1741   per_vrf_sessions_t *per_vrf_sessions;
1742   snat_main_per_thread_data_t *tsm;
1743   snat_main_t *sm = &snat_main;
1744
1745   /* *INDENT-OFF* */
1746   vec_foreach (tsm, sm->per_thread_data)
1747     {
1748       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1749         {
1750           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1751               (per_vrf_sessions->tx_fib_index == fib_index))
1752             {
1753               per_vrf_sessions->expired = 1;
1754             }
1755         }
1756     }
1757   /* *INDENT-ON* */
1758 }
1759
1760 void
1761 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1762 {
1763   snat_main_t *sm = &snat_main;
1764   nat_fib_t *fib;
1765
1766   // we don't care if it is outside/inside fib
1767   // we just care about their ref_count
1768   // if it reaches 0 sessions should expire
1769   // because the fib isn't valid for NAT anymore
1770
1771   vec_foreach (fib, sm->fibs)
1772   {
1773     if (fib->fib_index == fib_index)
1774       {
1775         if (is_del)
1776           {
1777             fib->ref_count--;
1778             if (!fib->ref_count)
1779               {
1780                 vec_del1 (sm->fibs, fib - sm->fibs);
1781                 expire_per_vrf_sessions (fib_index);
1782               }
1783             return;
1784           }
1785         else
1786           fib->ref_count++;
1787       }
1788   }
1789   if (!is_del)
1790     {
1791       vec_add2 (sm->fibs, fib, 1);
1792       fib->ref_count = 1;
1793       fib->fib_index = fib_index;
1794     }
1795 }
1796
1797 int
1798 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1799 {
1800   snat_main_t *sm = &snat_main;
1801   snat_interface_t *i;
1802   const char *feature_name, *del_feature_name;
1803   snat_address_t *ap;
1804   snat_static_mapping_t *m;
1805   nat_outside_fib_t *outside_fib;
1806   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1807                                                        sw_if_index);
1808
1809   if (!sm->enabled)
1810     {
1811       nat_log_err ("nat44 is disabled");
1812       return VNET_API_ERROR_UNSUPPORTED;
1813     }
1814
1815   if (sm->out2in_dpo && !is_inside)
1816     {
1817       nat_log_err ("error unsupported");
1818       return VNET_API_ERROR_UNSUPPORTED;
1819     }
1820
1821   /* *INDENT-OFF* */
1822   pool_foreach (i, sm->output_feature_interfaces)
1823    {
1824     if (i->sw_if_index == sw_if_index)
1825       {
1826         nat_log_err ("error interface already configured");
1827         return VNET_API_ERROR_VALUE_EXIST;
1828       }
1829   }
1830   /* *INDENT-ON* */
1831
1832   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1833     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1834   else
1835     {
1836       if (sm->num_workers > 1)
1837         feature_name =
1838           is_inside ? "nat44-in2out-worker-handoff" :
1839           "nat44-out2in-worker-handoff";
1840       else if (sm->endpoint_dependent)
1841         {
1842           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1843         }
1844       else
1845         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1846     }
1847
1848   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
1849     sm->fq_in2out_index =
1850       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1851
1852   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1853     sm->fq_out2in_index =
1854       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1855
1856   if (sm->endpoint_dependent)
1857     update_per_vrf_sessions_vec (fib_index, is_del);
1858
1859   if (!is_inside)
1860     {
1861       /* *INDENT-OFF* */
1862       vec_foreach (outside_fib, sm->outside_fibs)
1863         {
1864           if (outside_fib->fib_index == fib_index)
1865             {
1866               if (is_del)
1867                 {
1868                   outside_fib->refcount--;
1869                   if (!outside_fib->refcount)
1870                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1871                 }
1872               else
1873                 outside_fib->refcount++;
1874               goto feature_set;
1875             }
1876         }
1877       /* *INDENT-ON* */
1878       if (!is_del)
1879         {
1880           vec_add2 (sm->outside_fibs, outside_fib, 1);
1881           outside_fib->refcount = 1;
1882           outside_fib->fib_index = fib_index;
1883         }
1884     }
1885
1886 feature_set:
1887   /* *INDENT-OFF* */
1888   pool_foreach (i, sm->interfaces)
1889    {
1890     if (i->sw_if_index == sw_if_index)
1891       {
1892         if (is_del)
1893           {
1894             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1895               {
1896                 if (is_inside)
1897                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1898                 else
1899                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1900
1901                 if (sm->num_workers > 1)
1902                   {
1903                     del_feature_name = "nat44-handoff-classify";
1904                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1905                                                  "nat44-out2in-worker-handoff";
1906                   }
1907                 else if (sm->endpoint_dependent)
1908                   {
1909                     del_feature_name = "nat44-ed-classify";
1910                     feature_name = !is_inside ?  "nat-pre-in2out" :
1911                                                  "nat-pre-out2in";
1912                   }
1913                 else
1914                   {
1915                     del_feature_name = "nat44-classify";
1916                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1917                   }
1918
1919                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1920                 if (rv)
1921                   return rv;
1922                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1923                                              sw_if_index, 0, 0, 0);
1924                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1925                                              sw_if_index, 1, 0, 0);
1926                 if (!is_inside)
1927                   {
1928                     if (sm->endpoint_dependent)
1929                       vnet_feature_enable_disable ("ip4-local",
1930                                                    "nat44-ed-hairpinning",
1931                                                    sw_if_index, 1, 0, 0);
1932                     else
1933                       vnet_feature_enable_disable ("ip4-local",
1934                                                    "nat44-hairpinning",
1935                                                    sw_if_index, 1, 0, 0);
1936                   }
1937               }
1938             else
1939               {
1940                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1941                 if (rv)
1942                   return rv;
1943                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1944                                              sw_if_index, 0, 0, 0);
1945                 pool_put (sm->interfaces, i);
1946                 if (is_inside)
1947                   {
1948                     if (sm->endpoint_dependent)
1949                       vnet_feature_enable_disable ("ip4-local",
1950                                                    "nat44-ed-hairpinning",
1951                                                    sw_if_index, 0, 0, 0);
1952                     else
1953                       vnet_feature_enable_disable ("ip4-local",
1954                                                    "nat44-hairpinning",
1955                                                    sw_if_index, 0, 0, 0);
1956                   }
1957               }
1958           }
1959         else
1960           {
1961             if ((nat_interface_is_inside(i) && is_inside) ||
1962                 (nat_interface_is_outside(i) && !is_inside))
1963               return 0;
1964
1965             if (sm->num_workers > 1)
1966               {
1967                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1968                                                  "nat44-out2in-worker-handoff";
1969                 feature_name = "nat44-handoff-classify";
1970               }
1971             else if (sm->endpoint_dependent)
1972               {
1973                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
1974                                                  "nat-pre-out2in";
1975
1976                 feature_name = "nat44-ed-classify";
1977               }
1978             else
1979               {
1980                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1981                 feature_name = "nat44-classify";
1982               }
1983
1984             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1985             if (rv)
1986               return rv;
1987             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1988                                          sw_if_index, 0, 0, 0);
1989             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1990                                          sw_if_index, 1, 0, 0);
1991             if (!is_inside)
1992               {
1993                 if (sm->endpoint_dependent)
1994                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1995                                                sw_if_index, 0, 0, 0);
1996                 else
1997                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1998                                                sw_if_index, 0, 0, 0);
1999               }
2000             goto set_flags;
2001           }
2002
2003         goto fib;
2004       }
2005   }
2006   /* *INDENT-ON* */
2007
2008   if (is_del)
2009     {
2010       nat_log_err ("error interface couldn't be found");
2011       return VNET_API_ERROR_NO_SUCH_ENTRY;
2012     }
2013
2014   pool_get (sm->interfaces, i);
2015   i->sw_if_index = sw_if_index;
2016   i->flags = 0;
2017   nat_validate_counters (sm, sw_if_index);
2018
2019   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2020                                0);
2021
2022   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2023   if (rv)
2024     return rv;
2025
2026   if (is_inside && !sm->out2in_dpo)
2027     {
2028       if (sm->endpoint_dependent)
2029         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2030                                      sw_if_index, 1, 0, 0);
2031       else
2032         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2033                                      sw_if_index, 1, 0, 0);
2034     }
2035
2036 set_flags:
2037   if (is_inside)
2038     {
2039       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2040       return 0;
2041     }
2042   else
2043     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2044
2045   /* Add/delete external addresses to FIB */
2046 fib:
2047   /* *INDENT-OFF* */
2048   vec_foreach (ap, sm->addresses)
2049     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2050
2051   pool_foreach (m, sm->static_mappings)
2052    {
2053     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2054       continue;
2055
2056     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2057   }
2058   /* *INDENT-ON* */
2059
2060   return 0;
2061 }
2062
2063 int
2064 snat_interface_add_del_output_feature (u32 sw_if_index,
2065                                        u8 is_inside, int is_del)
2066 {
2067   snat_main_t *sm = &snat_main;
2068   snat_interface_t *i;
2069   snat_address_t *ap;
2070   snat_static_mapping_t *m;
2071   nat_outside_fib_t *outside_fib;
2072   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2073                                                        sw_if_index);
2074
2075   if (!sm->enabled)
2076     {
2077       nat_log_err ("nat44 is disabled");
2078       return VNET_API_ERROR_UNSUPPORTED;
2079     }
2080
2081   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2082     {
2083       nat_log_err ("error unsupported");
2084       return VNET_API_ERROR_UNSUPPORTED;
2085     }
2086
2087   /* *INDENT-OFF* */
2088   pool_foreach (i, sm->interfaces)
2089    {
2090     if (i->sw_if_index == sw_if_index)
2091       {
2092         nat_log_err ("error interface already configured");
2093         return VNET_API_ERROR_VALUE_EXIST;
2094       }
2095   }
2096   /* *INDENT-ON* */
2097
2098   if (sm->endpoint_dependent)
2099     update_per_vrf_sessions_vec (fib_index, is_del);
2100
2101   if (!is_inside)
2102     {
2103       /* *INDENT-OFF* */
2104       vec_foreach (outside_fib, sm->outside_fibs)
2105         {
2106           if (outside_fib->fib_index == fib_index)
2107             {
2108               if (is_del)
2109                 {
2110                   outside_fib->refcount--;
2111                   if (!outside_fib->refcount)
2112                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2113                 }
2114               else
2115                 outside_fib->refcount++;
2116               goto feature_set;
2117             }
2118         }
2119       /* *INDENT-ON* */
2120       if (!is_del)
2121         {
2122           vec_add2 (sm->outside_fibs, outside_fib, 1);
2123           outside_fib->refcount = 1;
2124           outside_fib->fib_index = fib_index;
2125         }
2126     }
2127
2128 feature_set:
2129   if (is_inside)
2130     {
2131       if (sm->endpoint_dependent)
2132         {
2133           int rv =
2134             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2135           if (rv)
2136             return rv;
2137           rv =
2138             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2139                                                             !is_del);
2140           if (rv)
2141             return rv;
2142           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2143                                        sw_if_index, !is_del, 0, 0);
2144           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2145                                        sw_if_index, !is_del, 0, 0);
2146         }
2147       else
2148         {
2149           int rv =
2150             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2151           if (rv)
2152             return rv;
2153           rv =
2154             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2155                                                             !is_del);
2156           if (rv)
2157             return rv;
2158           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2159                                        sw_if_index, !is_del, 0, 0);
2160           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2161                                        sw_if_index, !is_del, 0, 0);
2162         }
2163       goto fq;
2164     }
2165
2166   if (sm->num_workers > 1)
2167     {
2168       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2169       if (rv)
2170         return rv;
2171       rv =
2172         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2173       if (rv)
2174         return rv;
2175       vnet_feature_enable_disable ("ip4-unicast",
2176                                    "nat44-out2in-worker-handoff",
2177                                    sw_if_index, !is_del, 0, 0);
2178       vnet_feature_enable_disable ("ip4-output",
2179                                    "nat44-in2out-output-worker-handoff",
2180                                    sw_if_index, !is_del, 0, 0);
2181     }
2182   else
2183     {
2184       if (sm->endpoint_dependent)
2185         {
2186           int rv =
2187             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2188           if (rv)
2189             return rv;
2190           rv =
2191             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2192                                                             !is_del);
2193           if (rv)
2194             return rv;
2195           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2196                                        sw_if_index, !is_del, 0, 0);
2197           vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2198                                        sw_if_index, !is_del, 0, 0);
2199         }
2200       else
2201         {
2202           int rv =
2203             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2204           if (rv)
2205             return rv;
2206           rv =
2207             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2208                                                             !is_del);
2209           if (rv)
2210             return rv;
2211           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2212                                        sw_if_index, !is_del, 0, 0);
2213           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2214                                        sw_if_index, !is_del, 0, 0);
2215         }
2216     }
2217
2218 fq:
2219   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2220     sm->fq_in2out_output_index =
2221       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2222
2223   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2224     sm->fq_out2in_index =
2225       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2226
2227   /* *INDENT-OFF* */
2228   pool_foreach (i, sm->output_feature_interfaces)
2229    {
2230     if (i->sw_if_index == sw_if_index)
2231       {
2232         if (is_del)
2233           pool_put (sm->output_feature_interfaces, i);
2234         else
2235           return VNET_API_ERROR_VALUE_EXIST;
2236
2237         goto fib;
2238       }
2239   }
2240   /* *INDENT-ON* */
2241
2242   if (is_del)
2243     {
2244       nat_log_err ("error interface couldn't be found");
2245       return VNET_API_ERROR_NO_SUCH_ENTRY;
2246     }
2247
2248   pool_get (sm->output_feature_interfaces, i);
2249   i->sw_if_index = sw_if_index;
2250   i->flags = 0;
2251   nat_validate_counters (sm, sw_if_index);
2252   if (is_inside)
2253     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2254   else
2255     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2256
2257   /* Add/delete external addresses to FIB */
2258 fib:
2259   if (is_inside)
2260     return 0;
2261
2262   /* *INDENT-OFF* */
2263   vec_foreach (ap, sm->addresses)
2264     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2265
2266   pool_foreach (m, sm->static_mappings)
2267    {
2268     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2269       continue;
2270
2271     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2272   }
2273   /* *INDENT-ON* */
2274
2275   return 0;
2276 }
2277
2278 int
2279 snat_set_workers (uword * bitmap)
2280 {
2281   snat_main_t *sm = &snat_main;
2282   int i, j = 0;
2283
2284   if (sm->num_workers < 2)
2285     return VNET_API_ERROR_FEATURE_DISABLED;
2286
2287   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2288     return VNET_API_ERROR_INVALID_WORKER;
2289
2290   vec_free (sm->workers);
2291   /* *INDENT-OFF* */
2292   clib_bitmap_foreach (i, bitmap)
2293     {
2294       vec_add1(sm->workers, i);
2295       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2296       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2297       j++;
2298     }
2299   /* *INDENT-ON* */
2300
2301   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2302
2303   return 0;
2304 }
2305
2306 static void
2307 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2308                          u32 sw_if_index, u32 new_fib_index,
2309                          u32 old_fib_index)
2310 {
2311   snat_main_t *sm = &snat_main;
2312   nat_outside_fib_t *outside_fib;
2313   snat_interface_t *i;
2314   u8 is_add = 1;
2315   u8 match = 0;
2316
2317   if (!sm->enabled || (new_fib_index == old_fib_index)
2318       || (!vec_len (sm->outside_fibs)))
2319     {
2320       return;
2321     }
2322
2323   /* *INDENT-OFF* */
2324   pool_foreach (i, sm->interfaces)
2325      {
2326       if (i->sw_if_index == sw_if_index)
2327         {
2328           if (!(nat_interface_is_outside (i)))
2329             return;
2330           match = 1;
2331         }
2332     }
2333
2334   pool_foreach (i, sm->output_feature_interfaces)
2335      {
2336       if (i->sw_if_index == sw_if_index)
2337         {
2338           if (!(nat_interface_is_outside (i)))
2339             return;
2340           match = 1;
2341         }
2342     }
2343   /* *INDENT-ON* */
2344
2345   if (!match)
2346     return;
2347
2348   vec_foreach (outside_fib, sm->outside_fibs)
2349   {
2350     if (outside_fib->fib_index == old_fib_index)
2351       {
2352         outside_fib->refcount--;
2353         if (!outside_fib->refcount)
2354           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2355         break;
2356       }
2357   }
2358
2359   vec_foreach (outside_fib, sm->outside_fibs)
2360   {
2361     if (outside_fib->fib_index == new_fib_index)
2362       {
2363         outside_fib->refcount++;
2364         is_add = 0;
2365         break;
2366       }
2367   }
2368
2369   if (is_add)
2370     {
2371       vec_add2 (sm->outside_fibs, outside_fib, 1);
2372       outside_fib->refcount = 1;
2373       outside_fib->fib_index = new_fib_index;
2374     }
2375 }
2376
2377 static void
2378 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2379                          u32 sw_if_index, u32 new_fib_index,
2380                          u32 old_fib_index);
2381
2382 static void
2383 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2384                                        uword opaque,
2385                                        u32 sw_if_index,
2386                                        ip4_address_t * address,
2387                                        u32 address_length,
2388                                        u32 if_address_index, u32 is_delete);
2389
2390 static void
2391 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2392                                  uword opaque,
2393                                  u32 sw_if_index,
2394                                  ip4_address_t * address,
2395                                  u32 address_length,
2396                                  u32 if_address_index, u32 is_delete);
2397
2398 void
2399 test_key_calc_split ()
2400 {
2401   ip4_address_t l_addr;
2402   l_addr.as_u8[0] = 1;
2403   l_addr.as_u8[1] = 1;
2404   l_addr.as_u8[2] = 1;
2405   l_addr.as_u8[3] = 1;
2406   ip4_address_t r_addr;
2407   r_addr.as_u8[0] = 2;
2408   r_addr.as_u8[1] = 2;
2409   r_addr.as_u8[2] = 2;
2410   r_addr.as_u8[3] = 2;
2411   u16 l_port = 40001;
2412   u16 r_port = 40301;
2413   u8 proto = 9;
2414   u32 fib_index = 9000001;
2415   u32 thread_index = 3000000001;
2416   u32 session_index = 3000000221;
2417   clib_bihash_kv_16_8_t kv;
2418   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2419               thread_index, session_index);
2420   ip4_address_t l_addr2;
2421   ip4_address_t r_addr2;
2422   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2423   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2424   u16 l_port2 = 0;
2425   u16 r_port2 = 0;
2426   u8 proto2 = 0;
2427   u32 fib_index2 = 0;
2428   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2429                &r_port2);
2430   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2431   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2432   ASSERT (l_port == l_port2);
2433   ASSERT (r_port == r_port2);
2434   ASSERT (proto == proto2);
2435   ASSERT (fib_index == fib_index2);
2436   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2437   ASSERT (session_index == ed_value_get_session_index (&kv));
2438
2439   fib_index = 7001;
2440   proto = 5;
2441   nat_protocol_t proto3 = ~0;
2442   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2443   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2444   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2445   ASSERT (l_port == l_port2);
2446   ASSERT (proto == proto3);
2447   ASSERT (fib_index == fib_index2);
2448 }
2449
2450 static clib_error_t *
2451 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2452 {
2453   snat_main_t *sm = &snat_main;
2454   u32 fib_index;
2455
2456   if (sm->endpoint_dependent)
2457     {
2458       // TODO: consider removing all NAT interfaces
2459       if (!is_add)
2460         {
2461           fib_index = ip4_fib_index_from_table_id (table_id);
2462           if (fib_index != ~0)
2463             expire_per_vrf_sessions (fib_index);
2464         }
2465     }
2466   return 0;
2467 }
2468
2469 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2470
2471 void
2472 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2473 {
2474   vlib_node_t *node;
2475
2476   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2477   sm->ei_out2in_node_index = node->index;
2478   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2479   sm->ei_in2out_node_index = node->index;
2480   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2481   sm->ei_in2out_output_node_index = node->index;
2482
2483   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2484   sm->ed_out2in_node_index = node->index;
2485   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2486   sm->ed_in2out_node_index = node->index;
2487   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2488   sm->ed_in2out_output_node_index = node->index;
2489
2490   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2491   sm->error_node_index = node->index;
2492   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2493   sm->pre_in2out_node_index = node->index;
2494   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2495   sm->pre_out2in_node_index = node->index;
2496   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2497   sm->pre_in2out_node_index = node->index;
2498   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2499   sm->pre_out2in_node_index = node->index;
2500   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2501   sm->in2out_fast_node_index = node->index;
2502   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2503   sm->in2out_slowpath_node_index = node->index;
2504   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2505   sm->in2out_slowpath_output_node_index = node->index;
2506   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2507   sm->ed_in2out_slowpath_node_index = node->index;
2508   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2509   sm->out2in_fast_node_index = node->index;
2510   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2511   sm->ed_out2in_slowpath_node_index = node->index;
2512   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2513   sm->hairpinning_node_index = node->index;
2514   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2515   sm->hairpin_dst_node_index = node->index;
2516   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2517   sm->hairpin_src_node_index = node->index;
2518   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2519   sm->ed_hairpinning_node_index = node->index;
2520   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2521   sm->ed_hairpin_dst_node_index = node->index;
2522   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2523   sm->ed_hairpin_src_node_index = node->index;
2524 }
2525
2526 #define nat_init_simple_counter(c, n, sn) \
2527 do                                        \
2528   {                                       \
2529     c.name = n;                           \
2530     c.stat_segment_name = sn;             \
2531     vlib_validate_simple_counter (&c, 0); \
2532     vlib_zero_simple_counter (&c, 0);     \
2533   } while (0);
2534
2535 static clib_error_t *
2536 nat_init (vlib_main_t * vm)
2537 {
2538   snat_main_t *sm = &snat_main;
2539   vlib_thread_main_t *tm = vlib_get_thread_main ();
2540   vlib_thread_registration_t *tr;
2541   ip4_add_del_interface_address_callback_t cbi = { 0 };
2542   ip4_table_bind_callback_t cbt = { 0 };
2543   u32 i, num_threads = 0;
2544   uword *p, *bitmap = 0;
2545
2546   clib_memset (sm, 0, sizeof (*sm));
2547
2548   // required
2549   sm->vnet_main = vnet_get_main ();
2550   // convenience
2551   sm->ip4_main = &ip4_main;
2552   sm->api_main = vlibapi_get_main ();
2553   sm->ip4_lookup_main = &ip4_main.lookup_main;
2554
2555   // frame queue indices used for handoff
2556   sm->fq_out2in_index = ~0;
2557   sm->fq_in2out_index = ~0;
2558   sm->fq_in2out_output_index = ~0;
2559
2560   sm->log_level = SNAT_LOG_ERROR;
2561
2562   nat44_set_node_indexes (sm, vm);
2563   sm->log_class = vlib_log_register_class ("nat", 0);
2564   nat_ipfix_logging_init (vm);
2565
2566   nat_init_simple_counter (sm->total_users, "total-users",
2567                            "/nat44/total-users");
2568   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2569                            "/nat44/total-sessions");
2570   nat_init_simple_counter (sm->user_limit_reached, "user-limit-reached",
2571                            "/nat44/user-limit-reached");
2572
2573 #define _(x)                                            \
2574   sm->counters.fastpath.in2out.x.name = #x;             \
2575   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2576       "/nat44/in2out/fastpath/" #x;                     \
2577   sm->counters.slowpath.in2out.x.name = #x;             \
2578   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2579       "/nat44/in2out/slowpath/" #x;                     \
2580   sm->counters.fastpath.out2in.x.name = #x;             \
2581   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2582       "/nat44/out2in/fastpath/" #x;                     \
2583   sm->counters.slowpath.out2in.x.name = #x;             \
2584   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2585       "/nat44/out2in/slowpath/" #x;                     \
2586   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2587   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2588       "/nat44/ed/in2out/fastpath/" #x;                  \
2589   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2590   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2591       "/nat44/ed/in2out/slowpath/" #x;                  \
2592   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2593   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2594       "/nat44/ed/out2in/fastpath/" #x;                  \
2595   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2596   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2597       "/nat44/ed/out2in/slowpath/" #x;
2598   foreach_nat_counter;
2599 #undef _
2600   sm->counters.hairpinning.name = "hairpinning";
2601   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2602
2603   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2604   if (p)
2605     {
2606       tr = (vlib_thread_registration_t *) p[0];
2607       if (tr)
2608         {
2609           sm->num_workers = tr->count;
2610           sm->first_worker_index = tr->first_index;
2611         }
2612     }
2613   num_threads = tm->n_vlib_mains - 1;
2614   sm->port_per_thread = 0xffff - 1024;
2615   vec_validate (sm->per_thread_data, num_threads);
2616
2617   /* Use all available workers by default */
2618   if (sm->num_workers > 1)
2619     {
2620
2621       for (i = 0; i < sm->num_workers; i++)
2622         bitmap = clib_bitmap_set (bitmap, i, 1);
2623       snat_set_workers (bitmap);
2624       clib_bitmap_free (bitmap);
2625     }
2626   else
2627     sm->per_thread_data[0].snat_thread_index = 0;
2628
2629   /* callbacks to call when interface address changes. */
2630   cbi.function = snat_ip4_add_del_interface_address_cb;
2631   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2632   cbi.function = nat_ip4_add_del_addr_only_sm_cb;
2633   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2634
2635   /* callbacks to call when interface to table biding changes */
2636   cbt.function = snat_update_outside_fib;
2637   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2638
2639   sm->fib_src_low =
2640     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2641                          FIB_SOURCE_BH_SIMPLE);
2642   sm->fib_src_hi =
2643     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2644                          FIB_SOURCE_BH_SIMPLE);
2645
2646   /* used only by out2in-dpo feature */
2647   nat_dpo_module_init ();
2648
2649   nat_affinity_init (vm);
2650   nat_ha_init (vm, sm->num_workers, num_threads);
2651
2652   test_key_calc_split ();
2653   return nat44_api_hookup (vm);
2654 }
2655
2656 VLIB_INIT_FUNCTION (nat_init);
2657
2658 int
2659 nat44_plugin_enable (nat44_config_t c)
2660 {
2661   snat_main_t *sm = &snat_main;
2662   u32 static_mapping_buckets = 1024;
2663   u32 static_mapping_memory_size = 64 << 20;
2664
2665   if (sm->enabled)
2666     {
2667       nat_log_err ("nat44 is enabled");
2668       return 1;
2669     }
2670
2671   // c.static_mapping_only + c.connection_tracking
2672   //  - supported in NAT EI & NAT ED
2673   // c.out2in_dpo, c.static_mapping_only
2674   //  - supported in NAT EI
2675
2676   if (c.endpoint_dependent)
2677     {
2678       if ((c.static_mapping_only && !c.connection_tracking) || c.out2in_dpo)
2679         {
2680           nat_log_err ("unsupported combination of configuration");
2681           return 1;
2682         }
2683       if (c.users || c.user_sessions)
2684         {
2685           nat_log_err ("unsupported combination of configuration");
2686           return 1;
2687         }
2688     }
2689
2690   nat_reset_timeouts (&sm->timeouts);
2691
2692   // nat44 feature configuration
2693   sm->endpoint_dependent = c.endpoint_dependent;
2694   sm->static_mapping_only = c.static_mapping_only;
2695   sm->static_mapping_connection_tracking = c.connection_tracking;
2696   sm->out2in_dpo = c.out2in_dpo;
2697   sm->forwarding_enabled = 0;
2698   sm->mss_clamping = 0;
2699
2700   if (!c.users)
2701     c.users = 1024;
2702
2703   sm->max_users_per_thread = c.users;
2704   sm->user_buckets = nat_calc_bihash_buckets (c.users);
2705
2706   if (!c.sessions)
2707     c.sessions = 10 * 1024;
2708
2709   sm->max_translations_per_thread = c.sessions;
2710   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2711
2712   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2713   sm->max_translations_per_user
2714     = c.user_sessions ? c.user_sessions : sm->max_translations_per_thread;
2715
2716   sm->outside_vrf_id = c.outside_vrf;
2717   sm->outside_fib_index =
2718     fib_table_find_or_create_and_lock
2719     (FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2720
2721   sm->inside_vrf_id = c.inside_vrf;
2722   sm->inside_fib_index =
2723     fib_table_find_or_create_and_lock
2724     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2725
2726   if (c.endpoint_dependent)
2727     {
2728       sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2729       sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
2730       sm->out2in_node_index = sm->ed_out2in_node_index;
2731       sm->in2out_node_index = sm->ed_in2out_node_index;
2732       sm->in2out_output_node_index = sm->ed_in2out_output_node_index;
2733       sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
2734       sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
2735
2736       // try to move it into nat44_db_init,
2737       // consider static mapping requirements
2738       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
2739                              sm->translation_buckets, 0);
2740       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
2741                                           format_ed_session_kvp);
2742
2743       nat_affinity_enable ();
2744     }
2745   else
2746     {
2747       sm->worker_out2in_cb = nat44_ei_get_out2in_worker_index;
2748       sm->worker_in2out_cb = nat44_ei_get_in2out_worker_index;
2749
2750       sm->out2in_node_index = sm->ei_out2in_node_index;
2751       sm->in2out_node_index = sm->ei_in2out_node_index;
2752       sm->in2out_output_node_index = sm->ei_in2out_output_node_index;
2753       sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
2754       sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
2755
2756       nat44_ei_plugin_enable ();
2757     }
2758
2759   // c.static_mapping & c.connection_tracking require
2760   // session database
2761   if (!c.static_mapping_only
2762       || (c.static_mapping_only && c.connection_tracking))
2763     {
2764       snat_main_per_thread_data_t *tsm;
2765       /* *INDENT-OFF* */
2766       vec_foreach (tsm, sm->per_thread_data)
2767         {
2768           nat44_db_init (tsm);
2769         }
2770       /* *INDENT-ON* */
2771     }
2772   else
2773     {
2774       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2775       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2776     }
2777
2778   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
2779                         "static_mapping_by_local", static_mapping_buckets,
2780                         static_mapping_memory_size);
2781   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
2782                                      format_static_mapping_kvp);
2783
2784   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
2785                         "static_mapping_by_external",
2786                         static_mapping_buckets, static_mapping_memory_size);
2787   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
2788                                      format_static_mapping_kvp);
2789
2790   // last: reset counters
2791   vlib_zero_simple_counter (&sm->total_users, 0);
2792   vlib_zero_simple_counter (&sm->total_sessions, 0);
2793   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2794
2795   sm->enabled = 1;
2796   sm->rconfig = c;
2797
2798   return 0;
2799 }
2800
2801 void
2802 nat44_addresses_free (snat_address_t ** addresses)
2803 {
2804   snat_address_t *ap;
2805   /* *INDENT-OFF* */
2806   vec_foreach (ap, *addresses)
2807     {
2808     #define _(N, i, n, s) \
2809       vec_free (ap->busy_##n##_ports_per_thread);
2810       foreach_nat_protocol
2811     #undef _
2812     }
2813   /* *INDENT-ON* */
2814   vec_free (*addresses);
2815   *addresses = 0;
2816 }
2817
2818 int
2819 nat44_plugin_disable ()
2820 {
2821   snat_main_t *sm = &snat_main;
2822   snat_interface_t *i, *vec;
2823   int error = 0;
2824
2825   if (!sm->enabled)
2826     {
2827       nat_log_err ("nat44 is disabled");
2828       return 1;
2829     }
2830
2831   if (!sm->endpoint_dependent)
2832     {
2833       nat44_ei_plugin_disable ();
2834     }
2835
2836   // first unregister all nodes from interfaces
2837   vec = vec_dup (sm->interfaces);
2838   /* *INDENT-OFF* */
2839   vec_foreach (i, vec)
2840     {
2841       if (nat_interface_is_inside(i))
2842         error = snat_interface_add_del (i->sw_if_index, 1, 1);
2843       if (nat_interface_is_outside(i))
2844         error = snat_interface_add_del (i->sw_if_index, 0, 1);
2845
2846       if (error)
2847         {
2848           nat_log_err ("error occurred while removing interface %u",
2849                        i->sw_if_index);
2850         }
2851     }
2852   /* *INDENT-ON* */
2853   vec_free (vec);
2854   sm->interfaces = 0;
2855
2856   vec = vec_dup (sm->output_feature_interfaces);
2857   /* *INDENT-OFF* */
2858   vec_foreach (i, vec)
2859     {
2860       if (nat_interface_is_inside(i))
2861         error = snat_interface_add_del_output_feature (i->sw_if_index, 1, 1);
2862       if (nat_interface_is_outside(i))
2863         error = snat_interface_add_del_output_feature (i->sw_if_index, 0, 1);
2864
2865       if (error)
2866         {
2867           nat_log_err ("error occurred while removing interface %u",
2868                        i->sw_if_index);
2869         }
2870     }
2871   /* *INDENT-ON* */
2872   vec_free (vec);
2873   sm->output_feature_interfaces = 0;
2874
2875   vec_free (sm->max_translations_per_fib);
2876
2877   if (sm->endpoint_dependent)
2878     {
2879       nat_affinity_disable ();
2880       clib_bihash_free_16_8 (&sm->out2in_ed);
2881     }
2882
2883   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
2884   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
2885
2886   if (!sm->static_mapping_only ||
2887       (sm->static_mapping_only && sm->static_mapping_connection_tracking))
2888     {
2889       snat_main_per_thread_data_t *tsm;
2890      /* *INDENT-OFF* */
2891       vec_foreach (tsm, sm->per_thread_data)
2892         {
2893           nat44_db_free (tsm);
2894         }
2895       /* *INDENT-ON* */
2896     }
2897
2898   pool_free (sm->static_mappings);
2899
2900   nat44_addresses_free (&sm->addresses);
2901   nat44_addresses_free (&sm->twice_nat_addresses);
2902
2903
2904   vec_free (sm->to_resolve);
2905   vec_free (sm->auto_add_sw_if_indices);
2906   vec_free (sm->auto_add_sw_if_indices_twice_nat);
2907
2908   sm->to_resolve = 0;
2909   sm->auto_add_sw_if_indices = 0;
2910   sm->auto_add_sw_if_indices_twice_nat = 0;
2911
2912   sm->forwarding_enabled = 0;
2913
2914   sm->enabled = 0;
2915   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2916
2917   return 0;
2918 }
2919
2920 void
2921 snat_free_outside_address_and_port (snat_address_t * addresses,
2922                                     u32 thread_index,
2923                                     ip4_address_t * addr,
2924                                     u16 port, nat_protocol_t protocol)
2925 {
2926   snat_address_t *a;
2927   u32 address_index;
2928   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2929
2930   for (address_index = 0; address_index < vec_len (addresses);
2931        address_index++)
2932     {
2933       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2934         break;
2935     }
2936
2937   ASSERT (address_index < vec_len (addresses));
2938
2939   a = addresses + address_index;
2940
2941   switch (protocol)
2942     {
2943 #define _(N, i, n, s) \
2944     case NAT_PROTOCOL_##N: \
2945       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2946       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2947       a->busy_##n##_ports--; \
2948       a->busy_##n##_ports_per_thread[thread_index]--; \
2949       break;
2950       foreach_nat_protocol
2951 #undef _
2952     default:
2953       nat_elog_info ("unknown protocol");
2954       return;
2955     }
2956 }
2957
2958 int
2959 nat_set_outside_address_and_port (snat_address_t *addresses, u32 thread_index,
2960                                   ip4_address_t addr, u16 port,
2961                                   nat_protocol_t protocol)
2962 {
2963   snat_address_t *a = 0;
2964   u32 address_index;
2965   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2966
2967   for (address_index = 0; address_index < vec_len (addresses);
2968        address_index++)
2969     {
2970       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2971         continue;
2972
2973       a = addresses + address_index;
2974       switch (protocol)
2975         {
2976 #define _(N, j, n, s) \
2977         case NAT_PROTOCOL_##N: \
2978           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2979             return VNET_API_ERROR_INSTANCE_IN_USE; \
2980           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2981           a->busy_##n##_ports_per_thread[thread_index]++; \
2982           a->busy_##n##_ports++; \
2983           return 0;
2984           foreach_nat_protocol
2985 #undef _
2986         default:
2987           nat_elog_info ("unknown protocol");
2988           return 1;
2989         }
2990     }
2991
2992   return VNET_API_ERROR_NO_SUCH_ENTRY;
2993 }
2994
2995 int
2996 snat_static_mapping_match (snat_main_t * sm,
2997                            ip4_address_t match_addr,
2998                            u16 match_port,
2999                            u32 match_fib_index,
3000                            nat_protocol_t match_protocol,
3001                            ip4_address_t * mapping_addr,
3002                            u16 * mapping_port,
3003                            u32 * mapping_fib_index,
3004                            u8 by_external,
3005                            u8 * is_addr_only,
3006                            twice_nat_type_t * twice_nat,
3007                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
3008                            u8 * is_identity_nat, snat_static_mapping_t ** out)
3009 {
3010   clib_bihash_kv_8_8_t kv, value;
3011   clib_bihash_8_8_t *mapping_hash;
3012   snat_static_mapping_t *m;
3013   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
3014   nat44_lb_addr_port_t *local;
3015   u8 backend_index;
3016
3017   if (!by_external)
3018     {
3019       mapping_hash = &sm->static_mapping_by_local;
3020       init_nat_k (&kv, match_addr, match_port, match_fib_index,
3021                   match_protocol);
3022       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3023         {
3024           /* Try address only mapping */
3025           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
3026           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3027             return 1;
3028         }
3029     }
3030   else
3031     {
3032       mapping_hash = &sm->static_mapping_by_external;
3033       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
3034       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3035         {
3036           /* Try address only mapping */
3037           init_nat_k (&kv, match_addr, 0, 0, 0);
3038           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
3039             return 1;
3040         }
3041     }
3042
3043   m = pool_elt_at_index (sm->static_mappings, value.value);
3044
3045   if (by_external)
3046     {
3047       if (is_lb_static_mapping (m))
3048         {
3049           if (PREDICT_FALSE (lb != 0))
3050             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
3051           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
3052                                                           match_addr,
3053                                                           match_protocol,
3054                                                           match_port,
3055                                                           &backend_index))
3056             {
3057               local = pool_elt_at_index (m->locals, backend_index);
3058               *mapping_addr = local->addr;
3059               *mapping_port = local->port;
3060               *mapping_fib_index = local->fib_index;
3061               goto end;
3062             }
3063           // pick locals matching this worker
3064           if (PREDICT_FALSE (sm->num_workers > 1))
3065             {
3066               u32 thread_index = vlib_get_thread_index ();
3067               /* *INDENT-OFF* */
3068               pool_foreach_index (i, m->locals)
3069                {
3070                 local = pool_elt_at_index (m->locals, i);
3071
3072                 ip4_header_t ip = {
3073                   .src_address = local->addr,
3074                 };
3075
3076                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
3077                     thread_index)
3078                   {
3079                     vec_add1 (tmp, i);
3080                   }
3081               }
3082               /* *INDENT-ON* */
3083               ASSERT (vec_len (tmp) != 0);
3084             }
3085           else
3086             {
3087               /* *INDENT-OFF* */
3088               pool_foreach_index (i, m->locals)
3089                {
3090                 vec_add1 (tmp, i);
3091               }
3092               /* *INDENT-ON* */
3093             }
3094           hi = vec_len (tmp) - 1;
3095           local = pool_elt_at_index (m->locals, tmp[hi]);
3096           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
3097           while (lo < hi)
3098             {
3099               mid = ((hi - lo) >> 1) + lo;
3100               local = pool_elt_at_index (m->locals, tmp[mid]);
3101               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
3102             }
3103           local = pool_elt_at_index (m->locals, tmp[lo]);
3104           if (!(local->prefix >= rand))
3105             return 1;
3106           *mapping_addr = local->addr;
3107           *mapping_port = local->port;
3108           *mapping_fib_index = local->fib_index;
3109           if (m->affinity)
3110             {
3111               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
3112                                                 match_protocol, match_port,
3113                                                 tmp[lo], m->affinity,
3114                                                 m->affinity_per_service_list_head_index))
3115                 nat_elog_info ("create affinity record failed");
3116             }
3117           vec_free (tmp);
3118         }
3119       else
3120         {
3121           if (PREDICT_FALSE (lb != 0))
3122             *lb = NO_LB_NAT;
3123           *mapping_fib_index = m->fib_index;
3124           *mapping_addr = m->local_addr;
3125           /* Address only mapping doesn't change port */
3126           *mapping_port = is_addr_only_static_mapping (m) ? match_port
3127             : m->local_port;
3128         }
3129     }
3130   else
3131     {
3132       *mapping_addr = m->external_addr;
3133       /* Address only mapping doesn't change port */
3134       *mapping_port = is_addr_only_static_mapping (m) ? match_port
3135         : m->external_port;
3136       *mapping_fib_index = sm->outside_fib_index;
3137     }
3138
3139 end:
3140   if (PREDICT_FALSE (is_addr_only != 0))
3141     *is_addr_only = is_addr_only_static_mapping (m);
3142
3143   if (PREDICT_FALSE (twice_nat != 0))
3144     *twice_nat = m->twice_nat;
3145
3146   if (PREDICT_FALSE (is_identity_nat != 0))
3147     *is_identity_nat = is_identity_static_mapping (m);
3148
3149   if (out != 0)
3150     *out = m;
3151
3152   return 0;
3153 }
3154
3155 void
3156 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3157 {
3158   snat_main_t *sm = &snat_main;
3159   dpo_id_t dpo_v4 = DPO_INVALID;
3160   fib_prefix_t pfx = {
3161     .fp_proto = FIB_PROTOCOL_IP4,
3162     .fp_len = 32,
3163     .fp_addr.ip4.as_u32 = addr.as_u32,
3164   };
3165
3166   if (is_add)
3167     {
3168       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3169       fib_table_entry_special_dpo_add (0, &pfx, sm->fib_src_hi,
3170                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3171       dpo_reset (&dpo_v4);
3172     }
3173   else
3174     {
3175       fib_table_entry_special_remove (0, &pfx, sm->fib_src_hi);
3176     }
3177 }
3178
3179 static u32
3180 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3181                                u8 is_output)
3182 {
3183   snat_main_t *sm = &snat_main;
3184   u32 next_worker_index = sm->first_worker_index;
3185   u32 hash;
3186
3187   clib_bihash_kv_16_8_t kv16, value16;
3188   snat_main_per_thread_data_t *tsm;
3189   udp_header_t *udp;
3190
3191   if (PREDICT_FALSE (is_output))
3192     {
3193       u32 fib_index = sm->outside_fib_index;
3194       nat_outside_fib_t *outside_fib;
3195       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3196       fib_prefix_t pfx = {
3197         .fp_proto = FIB_PROTOCOL_IP4,
3198         .fp_len = 32,
3199         .fp_addr = {
3200                     .ip4.as_u32 = ip->dst_address.as_u32,
3201                     }
3202         ,
3203       };
3204
3205       udp = ip4_next_header (ip);
3206
3207       switch (vec_len (sm->outside_fibs))
3208         {
3209         case 0:
3210           fib_index = sm->outside_fib_index;
3211           break;
3212         case 1:
3213           fib_index = sm->outside_fibs[0].fib_index;
3214           break;
3215         default:
3216             /* *INDENT-OFF* */
3217             vec_foreach (outside_fib, sm->outside_fibs)
3218               {
3219                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3220                 if (FIB_NODE_INDEX_INVALID != fei)
3221                   {
3222                     if (fib_entry_get_resolving_interface (fei) != ~0)
3223                       {
3224                         fib_index = outside_fib->fib_index;
3225                         break;
3226                       }
3227                   }
3228               }
3229             /* *INDENT-ON* */
3230           break;
3231         }
3232
3233       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3234                  udp->dst_port, fib_index, ip->protocol);
3235
3236       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3237                                                   &kv16, &value16)))
3238         {
3239           tsm =
3240             vec_elt_at_index (sm->per_thread_data,
3241                               ed_value_get_thread_index (&value16));
3242           next_worker_index += tsm->thread_index;
3243
3244           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3245                                   next_worker_index, fib_index,
3246                                   clib_net_to_host_u32 (ip->
3247                                                         src_address.as_u32),
3248                                   clib_net_to_host_u32 (ip->
3249                                                         dst_address.as_u32));
3250
3251           return next_worker_index;
3252         }
3253     }
3254
3255   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3256     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3257
3258   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3259     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3260   else
3261     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3262
3263   if (PREDICT_TRUE (!is_output))
3264     {
3265       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3266                               next_worker_index, rx_fib_index,
3267                               clib_net_to_host_u32 (ip->src_address.as_u32),
3268                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3269     }
3270   else
3271     {
3272       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3273                               next_worker_index, rx_fib_index,
3274                               clib_net_to_host_u32 (ip->src_address.as_u32),
3275                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3276     }
3277
3278   return next_worker_index;
3279 }
3280
3281 static u32
3282 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3283                                u32 rx_fib_index, u8 is_output)
3284 {
3285   snat_main_t *sm = &snat_main;
3286   clib_bihash_kv_8_8_t kv, value;
3287   clib_bihash_kv_16_8_t kv16, value16;
3288   snat_main_per_thread_data_t *tsm;
3289
3290   u32 proto, next_worker_index = 0;
3291   udp_header_t *udp;
3292   u16 port;
3293   snat_static_mapping_t *m;
3294   u32 hash;
3295
3296   proto = ip_proto_to_nat_proto (ip->protocol);
3297
3298   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3299     {
3300       udp = ip4_next_header (ip);
3301
3302       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3303                  udp->src_port, rx_fib_index, ip->protocol);
3304
3305       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3306                                                   &kv16, &value16)))
3307         {
3308           tsm =
3309             vec_elt_at_index (sm->per_thread_data,
3310                               ed_value_get_thread_index (&value16));
3311           vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
3312             ed_value_get_session_index (&value16);
3313           next_worker_index = sm->first_worker_index + tsm->thread_index;
3314           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3315                                   next_worker_index, rx_fib_index,
3316                                   clib_net_to_host_u32 (ip->
3317                                                         src_address.as_u32),
3318                                   clib_net_to_host_u32 (ip->
3319                                                         dst_address.as_u32));
3320           return next_worker_index;
3321         }
3322     }
3323   else if (proto == NAT_PROTOCOL_ICMP)
3324     {
3325       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
3326         {
3327           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3328                                                       &kv16, &value16)))
3329             {
3330               tsm =
3331                 vec_elt_at_index (sm->per_thread_data,
3332                                   ed_value_get_thread_index (&value16));
3333               next_worker_index = sm->first_worker_index + tsm->thread_index;
3334               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3335                                       next_worker_index, rx_fib_index,
3336                                       clib_net_to_host_u32 (ip->
3337                                                             src_address.as_u32),
3338                                       clib_net_to_host_u32 (ip->
3339                                                             dst_address.as_u32));
3340               return next_worker_index;
3341             }
3342         }
3343     }
3344
3345   /* first try static mappings without port */
3346   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3347     {
3348       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3349       if (!clib_bihash_search_8_8
3350           (&sm->static_mapping_by_external, &kv, &value))
3351         {
3352           m = pool_elt_at_index (sm->static_mappings, value.value);
3353           next_worker_index = m->workers[0];
3354           goto done;
3355         }
3356     }
3357
3358   /* unknown protocol */
3359   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3360     {
3361       /* use current thread */
3362       next_worker_index = vlib_get_thread_index ();
3363       goto done;
3364     }
3365
3366   udp = ip4_next_header (ip);
3367   port = udp->dst_port;
3368
3369   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3370     {
3371       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3372       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3373       if (!icmp_type_is_error_message
3374           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3375         port = vnet_buffer (b)->ip.reass.l4_src_port;
3376       else
3377         {
3378           /* if error message, then it's not fragmented and we can access it */
3379           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3380           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3381           void *l4_header = ip4_next_header (inner_ip);
3382           switch (proto)
3383             {
3384             case NAT_PROTOCOL_ICMP:
3385               icmp = (icmp46_header_t *) l4_header;
3386               echo = (icmp_echo_header_t *) (icmp + 1);
3387               port = echo->identifier;
3388               break;
3389             case NAT_PROTOCOL_UDP:
3390             case NAT_PROTOCOL_TCP:
3391               port = ((tcp_udp_header_t *) l4_header)->src_port;
3392               break;
3393             default:
3394               next_worker_index = vlib_get_thread_index ();
3395               goto done;
3396             }
3397         }
3398     }
3399
3400   /* try static mappings with port */
3401   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3402     {
3403       init_nat_k (&kv, ip->dst_address, port, 0, proto);
3404       if (!clib_bihash_search_8_8
3405           (&sm->static_mapping_by_external, &kv, &value))
3406         {
3407           m = pool_elt_at_index (sm->static_mappings, value.value);
3408           if (!is_lb_static_mapping (m))
3409             {
3410               next_worker_index = m->workers[0];
3411               goto done;
3412             }
3413
3414           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3415             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3416
3417           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3418             next_worker_index =
3419               m->workers[hash & (_vec_len (m->workers) - 1)];
3420           else
3421             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3422           goto done;
3423         }
3424     }
3425
3426   /* worker by outside port */
3427   next_worker_index = sm->first_worker_index;
3428   next_worker_index +=
3429     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3430
3431 done:
3432   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3433                           clib_net_to_host_u32 (ip->src_address.as_u32),
3434                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3435   return next_worker_index;
3436 }
3437
3438 static u32
3439 nat_calc_bihash_buckets (u32 n_elts)
3440 {
3441   n_elts = n_elts / 2.5;
3442   u64 lower_pow2 = 1;
3443   while (lower_pow2 * 2 < n_elts)
3444     {
3445       lower_pow2 = 2 * lower_pow2;
3446     }
3447   u64 upper_pow2 = 2 * lower_pow2;
3448   if ((upper_pow2 - n_elts) < (n_elts - lower_pow2))
3449     {
3450       if (upper_pow2 <= UINT32_MAX)
3451         {
3452           return upper_pow2;
3453         }
3454     }
3455   return lower_pow2;
3456 }
3457
3458 u32
3459 nat44_get_max_session_limit ()
3460 {
3461   snat_main_t *sm = &snat_main;
3462   u32 max_limit = 0, len = 0;
3463
3464   for (; len < vec_len (sm->max_translations_per_fib); len++)
3465     {
3466       if (max_limit < sm->max_translations_per_fib[len])
3467         max_limit = sm->max_translations_per_fib[len];
3468     }
3469   return max_limit;
3470 }
3471
3472 int
3473 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3474 {
3475   snat_main_t *sm = &snat_main;
3476   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3477   u32 len = vec_len (sm->max_translations_per_fib);
3478
3479   if (len <= fib_index)
3480     {
3481       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3482
3483       for (; len < vec_len (sm->max_translations_per_fib); len++)
3484         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3485     }
3486
3487   sm->max_translations_per_fib[fib_index] = session_limit;
3488   return 0;
3489 }
3490
3491 int
3492 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3493 {
3494   snat_main_t *sm = &snat_main;
3495
3496   if (nat44_set_session_limit (session_limit, vrf_id))
3497     return 1;
3498   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3499
3500   sm->translation_buckets =
3501     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3502
3503   nat44_sessions_clear ();
3504   return 0;
3505 }
3506
3507 void
3508 nat44_db_init (snat_main_per_thread_data_t * tsm)
3509 {
3510   snat_main_t *sm = &snat_main;
3511
3512   pool_alloc (tsm->sessions, sm->max_translations_per_thread);
3513   pool_alloc (tsm->lru_pool, sm->max_translations_per_thread);
3514
3515   dlist_elt_t *head;
3516
3517   pool_get (tsm->lru_pool, head);
3518   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3519   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3520
3521   pool_get (tsm->lru_pool, head);
3522   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3523   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3524
3525   pool_get (tsm->lru_pool, head);
3526   tsm->udp_lru_head_index = head - tsm->lru_pool;
3527   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3528
3529   pool_get (tsm->lru_pool, head);
3530   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3531   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3532
3533   pool_get (tsm->lru_pool, head);
3534   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3535   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3536
3537   if (sm->endpoint_dependent)
3538     {
3539       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3540                              sm->translation_buckets, 0);
3541       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3542                                           format_ed_session_kvp);
3543       /*
3544          clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
3545          sm->translation_buckets, 0);
3546          clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
3547          format_ed_session_kvp); */
3548     }
3549   else
3550     {
3551       clib_bihash_init_8_8 (&tsm->in2out, "in2out", sm->translation_buckets,
3552                             0);
3553       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
3554       clib_bihash_init_8_8 (&tsm->out2in, "out2in", sm->translation_buckets,
3555                             0);
3556       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
3557     }
3558
3559   // TODO: ED nat is not using these
3560   // before removal large refactor required
3561   pool_alloc (tsm->list_pool, sm->max_translations_per_thread);
3562   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets, 0);
3563   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
3564 }
3565
3566 void
3567 nat44_db_free (snat_main_per_thread_data_t * tsm)
3568 {
3569   snat_main_t *sm = &snat_main;
3570
3571   pool_free (tsm->sessions);
3572   pool_free (tsm->lru_pool);
3573
3574   if (sm->endpoint_dependent)
3575     {
3576       clib_bihash_free_16_8 (&tsm->in2out_ed);
3577       vec_free (tsm->per_vrf_sessions_vec);
3578     }
3579   else
3580     {
3581       clib_bihash_free_8_8 (&tsm->in2out);
3582       clib_bihash_free_8_8 (&tsm->out2in);
3583     }
3584
3585   // TODO: resolve static mappings (put only to !ED)
3586   pool_free (tsm->users);
3587   pool_free (tsm->list_pool);
3588   clib_bihash_free_8_8 (&tsm->user_hash);
3589 }
3590
3591 void
3592 nat44_sessions_clear ()
3593 {
3594   snat_main_t *sm = &snat_main;
3595   snat_main_per_thread_data_t *tsm;
3596
3597   if (sm->endpoint_dependent)
3598     {
3599       clib_bihash_free_16_8 (&sm->out2in_ed);
3600       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
3601                              clib_max (1,
3602                                        sm->num_workers) *
3603                              sm->translation_buckets, 0);
3604       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
3605                                           format_ed_session_kvp);
3606     }
3607
3608   /* *INDENT-OFF* */
3609   vec_foreach (tsm, sm->per_thread_data)
3610     {
3611       nat44_db_free (tsm);
3612       nat44_db_init (tsm);
3613     }
3614   /* *INDENT-ON* */
3615
3616   vlib_zero_simple_counter (&sm->total_users, 0);
3617   vlib_zero_simple_counter (&sm->total_sessions, 0);
3618 }
3619
3620 static void
3621 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3622                                  uword opaque,
3623                                  u32 sw_if_index,
3624                                  ip4_address_t * address,
3625                                  u32 address_length,
3626                                  u32 if_address_index, u32 is_delete)
3627 {
3628   snat_main_t *sm = &snat_main;
3629   snat_static_map_resolve_t *rp;
3630   snat_static_mapping_t *m;
3631   clib_bihash_kv_8_8_t kv, value;
3632   int i, rv;
3633   ip4_address_t l_addr;
3634
3635   if (!sm->enabled)
3636     return;
3637
3638   for (i = 0; i < vec_len (sm->to_resolve); i++)
3639     {
3640       rp = sm->to_resolve + i;
3641       if (rp->addr_only == 0)
3642         continue;
3643       if (rp->sw_if_index == sw_if_index)
3644         goto match;
3645     }
3646
3647   return;
3648
3649 match:
3650   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
3651               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
3652   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3653     m = 0;
3654   else
3655     m = pool_elt_at_index (sm->static_mappings, value.value);
3656
3657   if (!is_delete)
3658     {
3659       /* Don't trip over lease renewal, static config */
3660       if (m)
3661         return;
3662     }
3663   else
3664     {
3665       if (!m)
3666         return;
3667     }
3668
3669   /* Indetity mapping? */
3670   if (rp->l_addr.as_u32 == 0)
3671     l_addr.as_u32 = address[0].as_u32;
3672   else
3673     l_addr.as_u32 = rp->l_addr.as_u32;
3674   /* Add the static mapping */
3675   rv = snat_add_static_mapping (l_addr,
3676                                 address[0],
3677                                 rp->l_port,
3678                                 rp->e_port,
3679                                 rp->vrf_id,
3680                                 rp->addr_only, ~0 /* sw_if_index */ ,
3681                                 rp->proto, !is_delete, rp->twice_nat,
3682                                 rp->out2in_only, rp->tag, rp->identity_nat,
3683                                 rp->pool_addr, rp->exact);
3684   if (rv)
3685     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
3686 }
3687
3688 static void
3689 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3690                                        uword opaque,
3691                                        u32 sw_if_index,
3692                                        ip4_address_t * address,
3693                                        u32 address_length,
3694                                        u32 if_address_index, u32 is_delete)
3695 {
3696   snat_main_t *sm = &snat_main;
3697   snat_static_map_resolve_t *rp;
3698   ip4_address_t l_addr;
3699   int i, j;
3700   int rv;
3701   u8 twice_nat = 0;
3702   snat_address_t *addresses = sm->addresses;
3703
3704   if (!sm->enabled)
3705     return;
3706
3707   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3708     {
3709       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3710         goto match;
3711     }
3712
3713   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3714     {
3715       twice_nat = 1;
3716       addresses = sm->twice_nat_addresses;
3717       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3718         goto match;
3719     }
3720
3721   return;
3722
3723 match:
3724   if (!is_delete)
3725     {
3726       /* Don't trip over lease renewal, static config */
3727       for (j = 0; j < vec_len (addresses); j++)
3728         if (addresses[j].addr.as_u32 == address->as_u32)
3729           return;
3730
3731       (void) snat_add_address (sm, address, ~0, twice_nat);
3732       /* Scan static map resolution vector */
3733       for (j = 0; j < vec_len (sm->to_resolve); j++)
3734         {
3735           rp = sm->to_resolve + j;
3736           if (rp->addr_only)
3737             continue;
3738           /* On this interface? */
3739           if (rp->sw_if_index == sw_if_index)
3740             {
3741               /* Indetity mapping? */
3742               if (rp->l_addr.as_u32 == 0)
3743                 l_addr.as_u32 = address[0].as_u32;
3744               else
3745                 l_addr.as_u32 = rp->l_addr.as_u32;
3746               /* Add the static mapping */
3747               rv = snat_add_static_mapping (l_addr,
3748                                             address[0],
3749                                             rp->l_port,
3750                                             rp->e_port,
3751                                             rp->vrf_id,
3752                                             rp->addr_only,
3753                                             ~0 /* sw_if_index */ ,
3754                                             rp->proto,
3755                                             rp->is_add, rp->twice_nat,
3756                                             rp->out2in_only, rp->tag,
3757                                             rp->identity_nat,
3758                                             rp->pool_addr, rp->exact);
3759               if (rv)
3760                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
3761                                     "i4", rv);
3762             }
3763         }
3764       return;
3765     }
3766   else
3767     {
3768       (void) snat_del_address (sm, address[0], 1, twice_nat);
3769       return;
3770     }
3771 }
3772
3773 int
3774 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3775                             u8 twice_nat)
3776 {
3777   ip4_main_t *ip4_main = sm->ip4_main;
3778   ip4_address_t *first_int_addr;
3779   snat_static_map_resolve_t *rp;
3780   u32 *indices_to_delete = 0;
3781   int i, j;
3782   u32 *auto_add_sw_if_indices =
3783     twice_nat ? sm->
3784     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3785
3786   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3787     );
3788
3789   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3790     {
3791       if (auto_add_sw_if_indices[i] == sw_if_index)
3792         {
3793           if (is_del)
3794             {
3795               /* if have address remove it */
3796               if (first_int_addr)
3797                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3798               else
3799                 {
3800                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3801                     {
3802                       rp = sm->to_resolve + j;
3803                       if (rp->sw_if_index == sw_if_index)
3804                         vec_add1 (indices_to_delete, j);
3805                     }
3806                   if (vec_len (indices_to_delete))
3807                     {
3808                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3809                         vec_del1 (sm->to_resolve, j);
3810                       vec_free (indices_to_delete);
3811                     }
3812                 }
3813               if (twice_nat)
3814                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3815               else
3816                 vec_del1 (sm->auto_add_sw_if_indices, i);
3817             }
3818           else
3819             return VNET_API_ERROR_VALUE_EXIST;
3820
3821           return 0;
3822         }
3823     }
3824
3825   if (is_del)
3826     return VNET_API_ERROR_NO_SUCH_ENTRY;
3827
3828   /* add to the auto-address list */
3829   if (twice_nat)
3830     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3831   else
3832     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3833
3834   /* If the address is already bound - or static - add it now */
3835   if (first_int_addr)
3836     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3837
3838   return 0;
3839 }
3840
3841 int
3842 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3843                    nat_protocol_t proto, u32 vrf_id, int is_in)
3844 {
3845   snat_main_per_thread_data_t *tsm;
3846   clib_bihash_kv_8_8_t kv, value;
3847   ip4_header_t ip;
3848   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3849   snat_session_t *s;
3850   clib_bihash_8_8_t *t;
3851
3852   if (sm->endpoint_dependent)
3853     return VNET_API_ERROR_UNSUPPORTED;
3854
3855   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3856   if (sm->num_workers > 1)
3857     tsm =
3858       vec_elt_at_index (sm->per_thread_data,
3859                         sm->worker_in2out_cb (&ip, fib_index, 0));
3860   else
3861     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3862
3863   init_nat_k (&kv, *addr, port, fib_index, proto);
3864   t = is_in ? &tsm->in2out : &tsm->out2in;
3865   if (!clib_bihash_search_8_8 (t, &kv, &value))
3866     {
3867       if (pool_is_free_index (tsm->sessions, value.value))
3868         return VNET_API_ERROR_UNSPECIFIED;
3869
3870       s = pool_elt_at_index (tsm->sessions, value.value);
3871       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3872       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3873       return 0;
3874     }
3875
3876   return VNET_API_ERROR_NO_SUCH_ENTRY;
3877 }
3878
3879 int
3880 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3881                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3882                       u32 vrf_id, int is_in)
3883 {
3884   ip4_header_t ip;
3885   clib_bihash_16_8_t *t;
3886   clib_bihash_kv_16_8_t kv, value;
3887   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3888   snat_session_t *s;
3889   snat_main_per_thread_data_t *tsm;
3890
3891   if (!sm->endpoint_dependent)
3892     return VNET_API_ERROR_FEATURE_DISABLED;
3893
3894   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3895   if (sm->num_workers > 1)
3896     tsm =
3897       vec_elt_at_index (sm->per_thread_data,
3898                         sm->worker_in2out_cb (&ip, fib_index, 0));
3899   else
3900     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3901
3902   t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
3903   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
3904   if (clib_bihash_search_16_8 (t, &kv, &value))
3905     {
3906       return VNET_API_ERROR_NO_SUCH_ENTRY;
3907     }
3908
3909   if (pool_is_free_index (tsm->sessions, value.value))
3910     return VNET_API_ERROR_UNSPECIFIED;
3911   s = pool_elt_at_index (tsm->sessions, value.value);
3912   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3913   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3914   return 0;
3915 }
3916
3917 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3918                                  vlib_node_runtime_t * node,
3919                                  vlib_frame_t * frame)
3920 {
3921   return 0;
3922 }
3923
3924 /* *INDENT-OFF* */
3925 VLIB_REGISTER_NODE (nat_default_node) = {
3926   .name = "nat-default",
3927   .vector_size = sizeof (u32),
3928   .format_trace = 0,
3929   .type = VLIB_NODE_TYPE_INTERNAL,
3930   .n_errors = 0,
3931   .n_next_nodes = NAT_N_NEXT,
3932   .next_nodes = {
3933     [NAT_NEXT_DROP] = "error-drop",
3934     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3935     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3936     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3937     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3938     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3939     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3940     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3941     [NAT_NEXT_OUT2IN_ED_HANDOFF] = "nat44-ed-out2in-handoff",
3942     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3943     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3944   },
3945 };
3946 /* *INDENT-ON* */
3947
3948 /*
3949  * fd.io coding-style-patch-verification: ON
3950  *
3951  * Local Variables:
3952  * eval: (c-set-style "gnu")
3953  * End:
3954  */