src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/lib/ipfix_logging.h>
  25 #include <nat/nat_inlines.h>
  26 #include <nat/nat44/inlines.h>
  27 #include <nat/nat_affinity.h>
  28 #include <nat/nat_syslog.h>
  29 #include <nat/nat_ha.h>
  30 #include <vnet/fib/fib_table.h>
  31 #include <vnet/fib/ip4_fib.h>
  32 #include <vnet/ip/reass/ip4_sv_reass.h>
  33 #include <vppinfra/bihash_16_8.h>
  34 #include <nat/nat44/ed_inlines.h>
  35
  36 #include <vpp/app/version.h>
  37
  38 snat_main_t snat_main;
  39
  40 fib_source_t nat_fib_src_hi;
  41 fib_source_t nat_fib_src_low;
  42
  43 /* *INDENT-OFF* */
  44 /* Hook up input features */
  45 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  46   .arc_name = "ip4-unicast",
  47   .node_name = "nat-pre-in2out",
  48   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  49                                "ip4-sv-reassembly-feature"),
  50 };
  51 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  52   .arc_name = "ip4-unicast",
  53   .node_name = "nat-pre-out2in",
  54   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  55                                "ip4-dhcp-client-detect",
  56                                "ip4-sv-reassembly-feature"),
  57 };
  58 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  59   .arc_name = "ip4-unicast",
  60   .node_name = "nat44-in2out-worker-handoff",
  61   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  62 };
  63 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  64   .arc_name = "ip4-unicast",
  65   .node_name = "nat44-out2in-worker-handoff",
  66   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  67                                "ip4-dhcp-client-detect"),
  68 };
  69 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  70   .arc_name = "ip4-unicast",
  71   .node_name = "nat44-in2out",
  72   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  73 };
  74 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  75   .arc_name = "ip4-unicast",
  76   .node_name = "nat44-out2in",
  77   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  78                                "ip4-dhcp-client-detect"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-classify",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat44-ed-in2out",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  89 };
  90 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  91   .arc_name = "ip4-unicast",
  92   .node_name = "nat44-ed-out2in",
  93   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  94                                "ip4-dhcp-client-detect"),
  95 };
  96 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-ed-classify",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 100 };
 101 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 102   .arc_name = "ip4-unicast",
 103   .node_name = "nat44-handoff-classify",
 104   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 105 };
 106 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 107   .arc_name = "ip4-unicast",
 108   .node_name = "nat44-in2out-fast",
 109   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 110 };
 111 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 112   .arc_name = "ip4-unicast",
 113   .node_name = "nat44-out2in-fast",
 114   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 115                                "ip4-dhcp-client-detect"),
 116 };
 117 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 118   .arc_name = "ip4-unicast",
 119   .node_name = "nat44-hairpin-dst",
 120   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 121 };
 122 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 123   .arc_name = "ip4-unicast",
 124   .node_name = "nat44-ed-hairpin-dst",
 125   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 126 };
 127
 128 /* Hook up output features */
 129 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 130   .arc_name = "ip4-output",
 131   .node_name = "nat44-in2out-output",
 132   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 133 };
 134 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 135   .arc_name = "ip4-output",
 136   .node_name = "nat44-in2out-output-worker-handoff",
 137   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 140   .arc_name = "ip4-output",
 141   .node_name = "nat44-hairpin-src",
 142   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 143 };
 144 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 145   .arc_name = "ip4-output",
 146   .node_name = "nat-pre-in2out-output",
 147   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 148   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 149 };
 150 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-ed-in2out-output",
 153   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 154   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 155 };
 156 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 157   .arc_name = "ip4-output",
 158   .node_name = "nat44-ed-hairpin-src",
 159   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 160   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 161 };
 162
 163 /* Hook up ip4-local features */
 164 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 165 {
 166   .arc_name = "ip4-local",
 167   .node_name = "nat44-hairpinning",
 168   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 169 };
 170 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 171 {
 172   .arc_name = "ip4-local",
 173   .node_name = "nat44-ed-hairpinning",
 174   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 175 };
 176
 177
 178 VLIB_PLUGIN_REGISTER () = {
 179     .version = VPP_BUILD_VER,
 180     .description = "Network Address Translation (NAT)",
 181 };
 182 /* *INDENT-ON* */
 183
 184 void
 185 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 186                        u8 is_ha)
 187 {
 188   clib_bihash_kv_8_8_t kv;
 189   u8 proto;
 190   u16 r_port, l_port;
 191   ip4_address_t *l_addr, *r_addr;
 192   u32 fib_index = 0;
 193   clib_bihash_kv_16_8_t ed_kv;
 194   snat_main_per_thread_data_t *tsm =
 195     vec_elt_at_index (sm->per_thread_data, thread_index);
 196
 197   if (is_ed_session (s))
 198     {
 199       per_vrf_sessions_unregister_session (s, thread_index);
 200     }
 201
 202   if (is_fwd_bypass_session (s))
 203     {
 204       if (snat_is_unk_proto_session (s))
 205         {
 206           init_ed_k (&ed_kv, s->in2out.addr, 0, s->ext_host_addr, 0, 0,
 207                      s->in2out.port);
 208         }
 209       else
 210         {
 211           l_port = s->in2out.port;
 212           r_port = s->ext_host_port;
 213           l_addr = &s->in2out.addr;
 214           r_addr = &s->ext_host_addr;
 215           proto = nat_proto_to_ip_proto (s->nat_proto);
 216           init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index,
 217                      proto);
 218         }
 219       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 220         nat_elog_warn ("in2out_ed key del failed");
 221       return;
 222     }
 223
 224   /* session lookup tables */
 225   if (is_ed_session (s))
 226     {
 227       if (is_affinity_sessions (s))
 228         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 229                              s->nat_proto, s->out2in.port);
 230       l_addr = &s->out2in.addr;
 231       r_addr = &s->ext_host_addr;
 232       fib_index = s->out2in.fib_index;
 233       if (snat_is_unk_proto_session (s))
 234         {
 235           proto = s->in2out.port;
 236           r_port = 0;
 237           l_port = 0;
 238         }
 239       else
 240         {
 241           proto = nat_proto_to_ip_proto (s->nat_proto);
 242           l_port = s->out2in.port;
 243           r_port = s->ext_host_port;
 244         }
 245       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 246       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 247         nat_elog_warn ("out2in_ed key del failed");
 248       l_addr = &s->in2out.addr;
 249       fib_index = s->in2out.fib_index;
 250       if (!snat_is_unk_proto_session (s))
 251         l_port = s->in2out.port;
 252       if (is_twice_nat_session (s))
 253         {
 254           r_addr = &s->ext_host_nat_addr;
 255           r_port = s->ext_host_nat_port;
 256         }
 257       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 258       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 259         nat_elog_warn ("in2out_ed key del failed");
 260
 261       if (!is_ha)
 262         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 263                                &s->in2out.addr, s->in2out.port,
 264                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 265                                &s->out2in.addr, s->out2in.port,
 266                                &s->ext_host_addr, s->ext_host_port,
 267                                s->nat_proto, is_twice_nat_session (s));
 268     }
 269   else
 270     {
 271       init_nat_i2o_k (&kv, s);
 272       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 273         nat_elog_warn ("in2out key del failed");
 274       init_nat_o2i_k (&kv, s);
 275       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 276         nat_elog_warn ("out2in key del failed");
 277
 278       if (!is_ha)
 279         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 280                                  &s->in2out.addr, s->in2out.port,
 281                                  &s->out2in.addr, s->out2in.port,
 282                                  s->nat_proto);
 283     }
 284
 285   if (snat_is_unk_proto_session (s))
 286     return;
 287
 288   if (!is_ha)
 289     {
 290       /* log NAT event */
 291       nat_ipfix_logging_nat44_ses_delete (thread_index,
 292                                           s->in2out.addr.as_u32,
 293                                           s->out2in.addr.as_u32,
 294                                           s->nat_proto,
 295                                           s->in2out.port,
 296                                           s->out2in.port,
 297                                           s->in2out.fib_index);
 298
 299       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 300                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 301                    thread_index);
 302     }
 303
 304   /* Twice NAT address and port for external host */
 305   if (is_twice_nat_session (s))
 306     {
 307       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 308                                           thread_index,
 309                                           &s->ext_host_nat_addr,
 310                                           s->ext_host_nat_port, s->nat_proto);
 311     }
 312
 313   if (snat_is_session_static (s))
 314     return;
 315
 316   snat_free_outside_address_and_port (sm->addresses, thread_index,
 317                                       &s->out2in.addr, s->out2in.port,
 318                                       s->nat_proto);
 319 }
 320
 321 void
 322 nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
 323                          u32 thread_index, u8 is_ha)
 324 {
 325   u8 proto;
 326   u16 r_port, l_port;
 327   ip4_address_t *l_addr, *r_addr;
 328   u32 fib_index;
 329   clib_bihash_kv_16_8_t ed_kv;
 330   snat_main_per_thread_data_t *tsm =
 331     vec_elt_at_index (sm->per_thread_data, thread_index);
 332
 333   if (is_fwd_bypass_session (s))
 334     {
 335       if (snat_is_unk_proto_session (s))
 336         {
 337           proto = s->in2out.port;
 338           r_port = 0;
 339           l_port = 0;
 340         }
 341       else
 342         {
 343           proto = nat_proto_to_ip_proto (s->nat_proto);
 344           l_port = s->in2out.port;
 345           r_port = s->ext_host_port;
 346         }
 347
 348       l_addr = &s->in2out.addr;
 349       r_addr = &s->ext_host_addr;
 350       fib_index = 0;
 351       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 352
 353       if (PREDICT_FALSE
 354           (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 355         nat_elog_warn ("in2out_ed key del failed");
 356       return;
 357     }
 358
 359   /* session lookup tables */
 360   if (is_affinity_sessions (s))
 361     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 362                          s->nat_proto, s->out2in.port);
 363   l_addr = &s->out2in.addr;
 364   r_addr = &s->ext_host_addr;
 365   fib_index = s->out2in.fib_index;
 366   if (snat_is_unk_proto_session (s))
 367     {
 368       proto = s->in2out.port;
 369       r_port = 0;
 370       l_port = 0;
 371     }
 372   else
 373     {
 374       proto = nat_proto_to_ip_proto (s->nat_proto);
 375       l_port = s->out2in.port;
 376       r_port = s->ext_host_port;
 377     }
 378   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 379
 380   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0)))
 381     nat_elog_warn ("out2in_ed key del failed");
 382
 383   l_addr = &s->in2out.addr;
 384   fib_index = s->in2out.fib_index;
 385
 386   if (!snat_is_unk_proto_session (s))
 387     l_port = s->in2out.port;
 388
 389   if (is_twice_nat_session (s))
 390     {
 391       r_addr = &s->ext_host_nat_addr;
 392       r_port = s->ext_host_nat_port;
 393     }
 394   init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 395
 396   if (PREDICT_FALSE (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0)))
 397     nat_elog_warn ("in2out_ed key del failed");
 398
 399   if (!is_ha)
 400     {
 401       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 402                              &s->in2out.addr, s->in2out.port,
 403                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 404                              &s->out2in.addr, s->out2in.port,
 405                              &s->ext_host_addr, s->ext_host_port,
 406                              s->nat_proto, is_twice_nat_session (s));
 407     }
 408
 409   if (snat_is_unk_proto_session (s))
 410     return;
 411
 412   if (!is_ha)
 413     {
 414       nat_ipfix_logging_nat44_ses_delete (thread_index,
 415                                           s->in2out.addr.as_u32,
 416                                           s->out2in.addr.as_u32,
 417                                           s->nat_proto,
 418                                           s->in2out.port,
 419                                           s->out2in.port,
 420                                           s->in2out.fib_index);
 421       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 422                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 423                    thread_index);
 424     }
 425
 426   /* Twice NAT address and port for external host */
 427   if (is_twice_nat_session (s))
 428     {
 429       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 430                                           thread_index,
 431                                           &s->ext_host_nat_addr,
 432                                           s->ext_host_nat_port, s->nat_proto);
 433     }
 434
 435   if (snat_is_session_static (s))
 436     return;
 437
 438   snat_free_outside_address_and_port (sm->addresses, thread_index,
 439                                       &s->out2in.addr, s->out2in.port,
 440                                       s->nat_proto);
 441 }
 442
 443
 444 snat_user_t *
 445 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 446                         u32 thread_index)
 447 {
 448   snat_user_t *u = 0;
 449   snat_user_key_t user_key;
 450   clib_bihash_kv_8_8_t kv, value;
 451   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 452   dlist_elt_t *per_user_list_head_elt;
 453
 454   user_key.addr.as_u32 = addr->as_u32;
 455   user_key.fib_index = fib_index;
 456   kv.key = user_key.as_u64;
 457
 458   /* Ever heard of the "user" = src ip4 address before? */
 459   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 460     {
 461       if (pool_elts (tsm->users) >= sm->max_users_per_thread)
 462         {
 463           vlib_increment_simple_counter (&sm->user_limit_reached,
 464                                          thread_index, 0, 1);
 465           nat_elog_warn ("maximum user limit reached");
 466           return NULL;
 467         }
 468       /* no, make a new one */
 469       pool_get (tsm->users, u);
 470       clib_memset (u, 0, sizeof (*u));
 471
 472       u->addr.as_u32 = addr->as_u32;
 473       u->fib_index = fib_index;
 474
 475       pool_get (tsm->list_pool, per_user_list_head_elt);
 476
 477       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 478         tsm->list_pool;
 479
 480       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 481
 482       kv.value = u - tsm->users;
 483
 484       /* add user */
 485       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 486         {
 487           nat_elog_warn ("user_hash key add failed");
 488           nat44_delete_user_with_no_session (sm, u, thread_index);
 489           return NULL;
 490         }
 491
 492       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 493                                pool_elts (tsm->users));
 494     }
 495   else
 496     {
 497       u = pool_elt_at_index (tsm->users, value.value);
 498     }
 499
 500   return u;
 501 }
 502
 503 snat_session_t *
 504 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 505                               u32 thread_index, f64 now)
 506 {
 507   snat_session_t *s;
 508   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 509   u32 oldest_per_user_translation_list_index, session_index;
 510   dlist_elt_t *oldest_per_user_translation_list_elt;
 511   dlist_elt_t *per_user_translation_list_elt;
 512
 513   /* Over quota? Recycle the least recently used translation */
 514   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 515     {
 516       oldest_per_user_translation_list_index =
 517         clib_dlist_remove_head (tsm->list_pool,
 518                                 u->sessions_per_user_list_head_index);
 519
 520       ASSERT (oldest_per_user_translation_list_index != ~0);
 521
 522       /* Add it back to the end of the LRU list */
 523       clib_dlist_addtail (tsm->list_pool,
 524                           u->sessions_per_user_list_head_index,
 525                           oldest_per_user_translation_list_index);
 526       /* Get the list element */
 527       oldest_per_user_translation_list_elt =
 528         pool_elt_at_index (tsm->list_pool,
 529                            oldest_per_user_translation_list_index);
 530
 531       /* Get the session index from the list element */
 532       session_index = oldest_per_user_translation_list_elt->value;
 533
 534       /* Get the session */
 535       s = pool_elt_at_index (tsm->sessions, session_index);
 536       nat_free_session_data (sm, s, thread_index, 0);
 537       if (snat_is_session_static (s))
 538         u->nstaticsessions--;
 539       else
 540         u->nsessions--;
 541       s->flags = 0;
 542       s->total_bytes = 0;
 543       s->total_pkts = 0;
 544       s->state = 0;
 545       s->ext_host_addr.as_u32 = 0;
 546       s->ext_host_port = 0;
 547       s->ext_host_nat_addr.as_u32 = 0;
 548       s->ext_host_nat_port = 0;
 549     }
 550   else
 551     {
 552       pool_get (tsm->sessions, s);
 553       clib_memset (s, 0, sizeof (*s));
 554
 555       /* Create list elts */
 556       pool_get (tsm->list_pool, per_user_translation_list_elt);
 557       clib_dlist_init (tsm->list_pool,
 558                        per_user_translation_list_elt - tsm->list_pool);
 559
 560       per_user_translation_list_elt->value = s - tsm->sessions;
 561       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 562       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 563
 564       clib_dlist_addtail (tsm->list_pool,
 565                           s->per_user_list_head_index,
 566                           per_user_translation_list_elt - tsm->list_pool);
 567
 568       s->user_index = u - tsm->users;
 569       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 570                                pool_elts (tsm->sessions));
 571     }
 572
 573   s->ha_last_refreshed = now;
 574
 575   return s;
 576 }
 577
 578 void
 579 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 580                           int is_add)
 581 {
 582   fib_prefix_t prefix = {
 583     .fp_len = p_len,
 584     .fp_proto = FIB_PROTOCOL_IP4,
 585     .fp_addr = {
 586                 .ip4.as_u32 = addr->as_u32,
 587                 },
 588   };
 589   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 590
 591   if (is_add)
 592     fib_table_entry_update_one_path (fib_index,
 593                                      &prefix,
 594                                      nat_fib_src_low,
 595                                      (FIB_ENTRY_FLAG_CONNECTED |
 596                                       FIB_ENTRY_FLAG_LOCAL |
 597                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 598                                      DPO_PROTO_IP4,
 599                                      NULL,
 600                                      sw_if_index,
 601                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 602   else
 603     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 604 }
 605
 606 int
 607 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 608                   u8 twice_nat)
 609 {
 610   snat_address_t *ap;
 611   snat_interface_t *i;
 612   vlib_thread_main_t *tm = vlib_get_thread_main ();
 613
 614   if (twice_nat && !sm->endpoint_dependent)
 615     return VNET_API_ERROR_FEATURE_DISABLED;
 616
 617   /* Check if address already exists */
 618   /* *INDENT-OFF* */
 619   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 620     {
 621       if (ap->addr.as_u32 == addr->as_u32)
 622         return VNET_API_ERROR_VALUE_EXIST;
 623     }
 624   /* *INDENT-ON* */
 625
 626   if (twice_nat)
 627     vec_add2 (sm->twice_nat_addresses, ap, 1);
 628   else
 629     vec_add2 (sm->addresses, ap, 1);
 630
 631   ap->addr = *addr;
 632   if (vrf_id != ~0)
 633     ap->fib_index =
 634       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 635                                          nat_fib_src_low);
 636   else
 637     ap->fib_index = ~0;
 638 #define _(N, i, n, s) \
 639   clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 640   ap->busy_##n##_ports = 0; \
 641   ap->busy_##n##_ports_per_thread = 0;\
 642   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 643   foreach_nat_protocol
 644 #undef _
 645     if (twice_nat)
 646     return 0;
 647
 648   /* Add external address to FIB */
 649   /* *INDENT-OFF* */
 650   pool_foreach (i, sm->interfaces,
 651   ({
 652     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 653       continue;
 654
 655     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 656     break;
 657   }));
 658   pool_foreach (i, sm->output_feature_interfaces,
 659   ({
 660     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 661       continue;
 662
 663     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 664     break;
 665   }));
 666   /* *INDENT-ON* */
 667
 668   return 0;
 669 }
 670
 671 static int
 672 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 673 {
 674   snat_static_mapping_t *m;
 675   /* *INDENT-OFF* */
 676   pool_foreach (m, sm->static_mappings,
 677   ({
 678       if (is_addr_only_static_mapping (m) ||
 679           is_out2in_only_static_mapping (m) ||
 680           is_identity_static_mapping (m))
 681         continue;
 682       if (m->external_addr.as_u32 == addr.as_u32)
 683         return 1;
 684   }));
 685   /* *INDENT-ON* */
 686
 687   return 0;
 688 }
 689
 690 static void
 691 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 692                                        ip4_address_t l_addr,
 693                                        u16 l_port,
 694                                        u32 sw_if_index,
 695                                        u16 e_port,
 696                                        u32 vrf_id,
 697                                        nat_protocol_t proto,
 698                                        int addr_only, int is_add, u8 * tag,
 699                                        int twice_nat, int out2in_only,
 700                                        int identity_nat,
 701                                        ip4_address_t pool_addr, int exact)
 702 {
 703   snat_static_map_resolve_t *rp;
 704
 705   vec_add2 (sm->to_resolve, rp, 1);
 706   rp->l_addr.as_u32 = l_addr.as_u32;
 707   rp->l_port = l_port;
 708   rp->sw_if_index = sw_if_index;
 709   rp->e_port = e_port;
 710   rp->vrf_id = vrf_id;
 711   rp->proto = proto;
 712   rp->addr_only = addr_only;
 713   rp->is_add = is_add;
 714   rp->twice_nat = twice_nat;
 715   rp->out2in_only = out2in_only;
 716   rp->identity_nat = identity_nat;
 717   rp->tag = vec_dup (tag);
 718   rp->pool_addr = pool_addr;
 719   rp->exact = exact;
 720 }
 721
 722 static u32
 723 get_thread_idx_by_port (u16 e_port)
 724 {
 725   snat_main_t *sm = &snat_main;
 726   u32 thread_idx = sm->num_workers;
 727   if (sm->num_workers > 1)
 728     {
 729       thread_idx =
 730         sm->first_worker_index +
 731         sm->workers[(e_port - 1024) / sm->port_per_thread];
 732     }
 733   return thread_idx;
 734 }
 735
 736 void
 737 snat_static_mapping_del_sessions (snat_main_t * sm,
 738                                   snat_main_per_thread_data_t * tsm,
 739                                   snat_user_key_t u_key, int addr_only,
 740                                   ip4_address_t e_addr, u16 e_port)
 741 {
 742   clib_bihash_kv_8_8_t kv, value;
 743   kv.key = u_key.as_u64;
 744   u64 user_index;
 745   dlist_elt_t *head, *elt;
 746   snat_user_t *u;
 747   snat_session_t *s;
 748   u32 elt_index, head_index, ses_index;
 749   if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 750     {
 751       user_index = value.value;
 752       u = pool_elt_at_index (tsm->users, user_index);
 753       if (u->nstaticsessions)
 754         {
 755           head_index = u->sessions_per_user_list_head_index;
 756           head = pool_elt_at_index (tsm->list_pool, head_index);
 757           elt_index = head->next;
 758           elt = pool_elt_at_index (tsm->list_pool, elt_index);
 759           ses_index = elt->value;
 760           while (ses_index != ~0)
 761             {
 762               s = pool_elt_at_index (tsm->sessions, ses_index);
 763               elt = pool_elt_at_index (tsm->list_pool, elt->next);
 764               ses_index = elt->value;
 765
 766               if (!addr_only)
 767                 {
 768                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 769                       (s->out2in.port != e_port))
 770                     continue;
 771                 }
 772
 773               if (is_lb_session (s))
 774                 continue;
 775
 776               if (!snat_is_session_static (s))
 777                 continue;
 778
 779               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 780               nat44_delete_session (sm, s, tsm - sm->per_thread_data);
 781
 782               if (!addr_only)
 783                 break;
 784             }
 785         }
 786     }
 787 }
 788
 789 void
 790 snat_ed_static_mapping_del_sessions (snat_main_t * sm,
 791                                      snat_main_per_thread_data_t * tsm,
 792                                      ip4_address_t l_addr,
 793                                      u16 l_port,
 794                                      u8 protocol,
 795                                      u32 fib_index, int addr_only,
 796                                      ip4_address_t e_addr, u16 e_port)
 797 {
 798   snat_session_t *s;
 799   u32 *indexes_to_free = NULL;
 800   /* *INDENT-OFF* */
 801   pool_foreach (s, tsm->sessions, {
 802     if (s->in2out.fib_index != fib_index ||
 803         s->in2out.addr.as_u32 != l_addr.as_u32)
 804       {
 805         continue;
 806       }
 807     if (!addr_only)
 808       {
 809         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 810             s->out2in.port != e_port ||
 811             s->in2out.port != l_port ||
 812             s->nat_proto != protocol)
 813           continue;
 814       }
 815
 816     if (is_lb_session (s))
 817       continue;
 818     if (!snat_is_session_static (s))
 819       continue;
 820     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 821     vec_add1 (indexes_to_free, s - tsm->sessions);
 822     if (!addr_only)
 823       break;
 824   });
 825   /* *INDENT-ON* */
 826   u32 *ses_index;
 827   vec_foreach (ses_index, indexes_to_free)
 828   {
 829     s = pool_elt_at_index (tsm->sessions, *ses_index);
 830     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 831   }
 832   vec_free (indexes_to_free);
 833 }
 834
 835 int
 836 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 837                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 838                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 839                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 840                          u8 identity_nat, ip4_address_t pool_addr, int exact)
 841 {
 842   snat_main_t *sm = &snat_main;
 843   snat_static_mapping_t *m;
 844   clib_bihash_kv_8_8_t kv, value;
 845   snat_address_t *a = 0;
 846   u32 fib_index = ~0;
 847   snat_interface_t *interface;
 848   int i;
 849   snat_main_per_thread_data_t *tsm;
 850   snat_user_key_t u_key;
 851   snat_user_t *u;
 852   dlist_elt_t *head, *elt;
 853   u32 elt_index, head_index;
 854   u32 ses_index;
 855   u64 user_index;
 856   snat_session_t *s;
 857   snat_static_map_resolve_t *rp, *rp_match = 0;
 858   nat44_lb_addr_port_t *local;
 859   u32 find = ~0;
 860
 861   if (!sm->endpoint_dependent)
 862     {
 863       if (twice_nat || out2in_only)
 864         return VNET_API_ERROR_FEATURE_DISABLED;
 865     }
 866
 867   /* If the external address is a specific interface address */
 868   if (sw_if_index != ~0)
 869     {
 870       ip4_address_t *first_int_addr;
 871
 872       for (i = 0; i < vec_len (sm->to_resolve); i++)
 873         {
 874           rp = sm->to_resolve + i;
 875           if (rp->sw_if_index != sw_if_index ||
 876               rp->l_addr.as_u32 != l_addr.as_u32 ||
 877               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 878             continue;
 879
 880           if (!addr_only)
 881             {
 882               if ((rp->l_port != l_port && rp->e_port != e_port)
 883                   || rp->proto != proto)
 884                 continue;
 885             }
 886
 887           rp_match = rp;
 888           break;
 889         }
 890
 891       /* Might be already set... */
 892       first_int_addr = ip4_interface_first_address
 893         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 894
 895       if (is_add)
 896         {
 897           if (rp_match)
 898             return VNET_API_ERROR_VALUE_EXIST;
 899
 900           snat_add_static_mapping_when_resolved
 901             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 902              addr_only, is_add, tag, twice_nat, out2in_only,
 903              identity_nat, pool_addr, exact);
 904
 905           /* DHCP resolution required? */
 906           if (first_int_addr == 0)
 907             {
 908               return 0;
 909             }
 910           else
 911             {
 912               e_addr.as_u32 = first_int_addr->as_u32;
 913               /* Identity mapping? */
 914               if (l_addr.as_u32 == 0)
 915                 l_addr.as_u32 = e_addr.as_u32;
 916             }
 917         }
 918       else
 919         {
 920           if (!rp_match)
 921             return VNET_API_ERROR_NO_SUCH_ENTRY;
 922
 923           vec_del1 (sm->to_resolve, i);
 924
 925           if (first_int_addr)
 926             {
 927               e_addr.as_u32 = first_int_addr->as_u32;
 928               /* Identity mapping? */
 929               if (l_addr.as_u32 == 0)
 930                 l_addr.as_u32 = e_addr.as_u32;
 931             }
 932           else
 933             return 0;
 934         }
 935     }
 936
 937   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 938   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 939     m = 0;
 940   else
 941     m = pool_elt_at_index (sm->static_mappings, value.value);
 942
 943   if (is_add)
 944     {
 945       if (m)
 946         {
 947           if (is_identity_static_mapping (m))
 948             {
 949               /* *INDENT-OFF* */
 950               pool_foreach (local, m->locals,
 951               ({
 952                 if (local->vrf_id == vrf_id)
 953                   return VNET_API_ERROR_VALUE_EXIST;
 954               }));
 955               /* *INDENT-ON* */
 956               pool_get (m->locals, local);
 957               local->vrf_id = vrf_id;
 958               local->fib_index =
 959                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 960                                                    nat_fib_src_low);
 961               init_nat_kv (&kv, m->local_addr, m->local_port,
 962                            local->fib_index, m->proto,
 963                            m - sm->static_mappings);
 964               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 965               return 0;
 966             }
 967           else
 968             return VNET_API_ERROR_VALUE_EXIST;
 969         }
 970
 971       if (twice_nat && addr_only)
 972         return VNET_API_ERROR_UNSUPPORTED;
 973
 974       /* Convert VRF id to FIB index */
 975       if (vrf_id != ~0)
 976         fib_index =
 977           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 978                                              nat_fib_src_low);
 979       /* If not specified use inside VRF id from SNAT plugin startup config */
 980       else
 981         {
 982           fib_index = sm->inside_fib_index;
 983           vrf_id = sm->inside_vrf_id;
 984           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 985         }
 986
 987       if (!(out2in_only || identity_nat))
 988         {
 989           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
 990                       addr_only ? 0 : proto);
 991           if (!clib_bihash_search_8_8
 992               (&sm->static_mapping_by_local, &kv, &value))
 993             return VNET_API_ERROR_VALUE_EXIST;
 994         }
 995
 996       /* Find external address in allocated addresses and reserve port for
 997          address and port pair mapping when dynamic translations enabled */
 998       if (!(addr_only || sm->static_mapping_only || out2in_only))
 999         {
1000           for (i = 0; i < vec_len (sm->addresses); i++)
1001             {
1002               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1003                 {
1004                   a = sm->addresses + i;
1005                   /* External port must be unused */
1006                   switch (proto)
1007                     {
1008 #define _(N, j, n, s) \
1009                     case NAT_PROTOCOL_##N: \
1010                       if (a->busy_##n##_port_refcounts[e_port]) \
1011                         return VNET_API_ERROR_INVALID_VALUE; \
1012                       ++a->busy_##n##_port_refcounts[e_port]; \
1013                       if (e_port > 1024) \
1014                         { \
1015                           a->busy_##n##_ports++; \
1016                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1017                         } \
1018                       break;
1019                       foreach_nat_protocol
1020 #undef _
1021                     default:
1022                       nat_elog_info ("unknown protocol");
1023                       return VNET_API_ERROR_INVALID_VALUE_2;
1024                     }
1025                   break;
1026                 }
1027             }
1028           /* External address must be allocated */
1029           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1030             {
1031               if (sw_if_index != ~0)
1032                 {
1033                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1034                     {
1035                       rp = sm->to_resolve + i;
1036                       if (rp->addr_only)
1037                         continue;
1038                       if (rp->sw_if_index != sw_if_index &&
1039                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1040                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1041                           rp->e_port != e_port && rp->proto != proto)
1042                         continue;
1043
1044                       vec_del1 (sm->to_resolve, i);
1045                       break;
1046                     }
1047                 }
1048               return VNET_API_ERROR_NO_SUCH_ENTRY;
1049             }
1050         }
1051
1052       pool_get (sm->static_mappings, m);
1053       clib_memset (m, 0, sizeof (*m));
1054       m->tag = vec_dup (tag);
1055       m->local_addr = l_addr;
1056       m->external_addr = e_addr;
1057       m->twice_nat = twice_nat;
1058
1059       if (twice_nat == TWICE_NAT && exact)
1060         {
1061           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
1062           m->pool_addr = pool_addr;
1063         }
1064
1065       if (out2in_only)
1066         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1067       if (addr_only)
1068         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
1069       if (identity_nat)
1070         {
1071           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
1072           pool_get (m->locals, local);
1073           local->vrf_id = vrf_id;
1074           local->fib_index = fib_index;
1075         }
1076       else
1077         {
1078           m->vrf_id = vrf_id;
1079           m->fib_index = fib_index;
1080         }
1081       if (!addr_only)
1082         {
1083           m->local_port = l_port;
1084           m->external_port = e_port;
1085           m->proto = proto;
1086         }
1087
1088       if (sm->num_workers > 1)
1089         {
1090           ip4_header_t ip = {
1091             .src_address = m->local_addr,
1092           };
1093           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
1094           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1095         }
1096       else
1097         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1098
1099       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
1100                    m - sm->static_mappings);
1101       if (!out2in_only)
1102         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1103
1104       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1105                    m - sm->static_mappings);
1106       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
1107
1108       /* Delete dynamic sessions matching local address (+ local port) */
1109       if (!(sm->static_mapping_only))
1110         {
1111           u_key.addr = m->local_addr;
1112           u_key.fib_index = m->fib_index;
1113           kv.key = u_key.as_u64;
1114           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1115             {
1116               user_index = value.value;
1117               u = pool_elt_at_index (tsm->users, user_index);
1118               if (u->nsessions)
1119                 {
1120                   head_index = u->sessions_per_user_list_head_index;
1121                   head = pool_elt_at_index (tsm->list_pool, head_index);
1122                   elt_index = head->next;
1123                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1124                   ses_index = elt->value;
1125                   while (ses_index != ~0)
1126                     {
1127                       s = pool_elt_at_index (tsm->sessions, ses_index);
1128                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1129                       ses_index = elt->value;
1130
1131                       if (snat_is_session_static (s))
1132                         continue;
1133
1134                       if (!addr_only && s->in2out.port != m->local_port)
1135                         continue;
1136
1137                       nat_free_session_data (sm, s,
1138                                              tsm - sm->per_thread_data, 0);
1139                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1140
1141                       if (!addr_only && !sm->endpoint_dependent)
1142                         break;
1143                     }
1144                 }
1145             }
1146         }
1147     }
1148   else
1149     {
1150       if (!m)
1151         {
1152           if (sw_if_index != ~0)
1153             return 0;
1154           else
1155             return VNET_API_ERROR_NO_SUCH_ENTRY;
1156         }
1157
1158       if (identity_nat)
1159         {
1160           if (vrf_id == ~0)
1161             vrf_id = sm->inside_vrf_id;
1162
1163           /* *INDENT-OFF* */
1164           pool_foreach (local, m->locals,
1165           ({
1166             if (local->vrf_id == vrf_id)
1167               find = local - m->locals;
1168           }));
1169           /* *INDENT-ON* */
1170           if (find == ~0)
1171             return VNET_API_ERROR_NO_SUCH_ENTRY;
1172
1173           local = pool_elt_at_index (m->locals, find);
1174           fib_index = local->fib_index;
1175           pool_put (m->locals, local);
1176         }
1177       else
1178         fib_index = m->fib_index;
1179
1180       /* Free external address port */
1181       if (!(addr_only || sm->static_mapping_only || out2in_only))
1182         {
1183           for (i = 0; i < vec_len (sm->addresses); i++)
1184             {
1185               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1186                 {
1187                   a = sm->addresses + i;
1188                   switch (proto)
1189                     {
1190 #define _(N, j, n, s) \
1191                     case NAT_PROTOCOL_##N: \
1192                       --a->busy_##n##_port_refcounts[e_port]; \
1193                       if (e_port > 1024) \
1194                         { \
1195                           a->busy_##n##_ports--; \
1196                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1197                         } \
1198                       break;
1199                       foreach_nat_protocol
1200 #undef _
1201                     default:
1202                       nat_elog_info ("unknown protocol");
1203                       return VNET_API_ERROR_INVALID_VALUE_2;
1204                     }
1205                   break;
1206                 }
1207             }
1208         }
1209
1210       if (sm->num_workers > 1)
1211         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1212       else
1213         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1214
1215       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
1216       if (!out2in_only)
1217         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1218
1219       /* Delete session(s) for static mapping if exist */
1220       if (!(sm->static_mapping_only) ||
1221           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1222         {
1223           if (sm->endpoint_dependent)
1224             {
1225               snat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr,
1226                                                    m->local_port, m->proto,
1227                                                    fib_index, addr_only,
1228                                                    e_addr, e_port);
1229             }
1230           else
1231             {
1232               u_key.addr = m->local_addr;
1233               u_key.fib_index = fib_index;
1234               kv.key = u_key.as_u64;
1235               snat_static_mapping_del_sessions (sm, tsm, u_key, addr_only,
1236                                                 e_addr, e_port);
1237             }
1238         }
1239
1240       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1241       if (pool_elts (m->locals))
1242         return 0;
1243
1244       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1245       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1246
1247       vec_free (m->tag);
1248       vec_free (m->workers);
1249       /* Delete static mapping from pool */
1250       pool_put (sm->static_mappings, m);
1251     }
1252
1253   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1254     return 0;
1255
1256   /* Add/delete external address to FIB */
1257   /* *INDENT-OFF* */
1258   pool_foreach (interface, sm->interfaces,
1259   ({
1260     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1261       continue;
1262
1263     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1264     break;
1265   }));
1266   pool_foreach (interface, sm->output_feature_interfaces,
1267   ({
1268     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1269       continue;
1270
1271     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1272     break;
1273   }));
1274   /* *INDENT-ON* */
1275
1276   return 0;
1277 }
1278
1279 int
1280 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1281                                  nat_protocol_t proto,
1282                                  nat44_lb_addr_port_t * locals, u8 is_add,
1283                                  twice_nat_type_t twice_nat, u8 out2in_only,
1284                                  u8 * tag, u32 affinity)
1285 {
1286   snat_main_t *sm = &snat_main;
1287   snat_static_mapping_t *m;
1288   clib_bihash_kv_8_8_t kv, value;
1289   snat_address_t *a = 0;
1290   int i;
1291   nat44_lb_addr_port_t *local;
1292   snat_main_per_thread_data_t *tsm;
1293   snat_session_t *s;
1294   uword *bitmap = 0;
1295
1296   if (!sm->endpoint_dependent)
1297     return VNET_API_ERROR_FEATURE_DISABLED;
1298
1299   init_nat_k (&kv, e_addr, e_port, 0, proto);
1300   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1301     m = 0;
1302   else
1303     m = pool_elt_at_index (sm->static_mappings, value.value);
1304
1305   if (is_add)
1306     {
1307       if (m)
1308         return VNET_API_ERROR_VALUE_EXIST;
1309
1310       if (vec_len (locals) < 2)
1311         return VNET_API_ERROR_INVALID_VALUE;
1312
1313       /* Find external address in allocated addresses and reserve port for
1314          address and port pair mapping when dynamic translations enabled */
1315       if (!(sm->static_mapping_only || out2in_only))
1316         {
1317           for (i = 0; i < vec_len (sm->addresses); i++)
1318             {
1319               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1320                 {
1321                   a = sm->addresses + i;
1322                   /* External port must be unused */
1323                   switch (proto)
1324                     {
1325 #define _(N, j, n, s) \
1326                     case NAT_PROTOCOL_##N: \
1327                       if (a->busy_##n##_port_refcounts[e_port]) \
1328                         return VNET_API_ERROR_INVALID_VALUE; \
1329                       ++a->busy_##n##_port_refcounts[e_port]; \
1330                       if (e_port > 1024) \
1331                         { \
1332                           a->busy_##n##_ports++; \
1333                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1334                         } \
1335                       break;
1336                       foreach_nat_protocol
1337 #undef _
1338                     default:
1339                       nat_elog_info ("unknown protocol");
1340                       return VNET_API_ERROR_INVALID_VALUE_2;
1341                     }
1342                   break;
1343                 }
1344             }
1345           /* External address must be allocated */
1346           if (!a)
1347             return VNET_API_ERROR_NO_SUCH_ENTRY;
1348         }
1349
1350       pool_get (sm->static_mappings, m);
1351       clib_memset (m, 0, sizeof (*m));
1352       m->tag = vec_dup (tag);
1353       m->external_addr = e_addr;
1354       m->external_port = e_port;
1355       m->proto = proto;
1356       m->twice_nat = twice_nat;
1357       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1358       if (out2in_only)
1359         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1360       m->affinity = affinity;
1361
1362       if (affinity)
1363         m->affinity_per_service_list_head_index =
1364           nat_affinity_get_per_service_list_head_index ();
1365       else
1366         m->affinity_per_service_list_head_index = ~0;
1367
1368       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto,
1369                    m - sm->static_mappings);
1370       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1371         {
1372           nat_elog_err ("static_mapping_by_external key add failed");
1373           return VNET_API_ERROR_UNSPECIFIED;
1374         }
1375
1376       for (i = 0; i < vec_len (locals); i++)
1377         {
1378           locals[i].fib_index =
1379             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1380                                                locals[i].vrf_id,
1381                                                nat_fib_src_low);
1382           if (!out2in_only)
1383             {
1384               init_nat_kv (&kv, locals[i].addr, locals[i].port,
1385                            locals[i].fib_index, m->proto,
1386                            m - sm->static_mappings);
1387               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1388             }
1389           locals[i].prefix = (i == 0) ? locals[i].probability :
1390             (locals[i - 1].prefix + locals[i].probability);
1391           pool_get (m->locals, local);
1392           *local = locals[i];
1393           if (sm->num_workers > 1)
1394             {
1395               ip4_header_t ip = {
1396                 .src_address = locals[i].addr,
1397               };
1398               bitmap =
1399                 clib_bitmap_set (bitmap,
1400                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1401                                  1);
1402             }
1403         }
1404
1405       /* Assign workers */
1406       if (sm->num_workers > 1)
1407         {
1408           /* *INDENT-OFF* */
1409           clib_bitmap_foreach (i, bitmap,
1410             ({
1411                vec_add1(m->workers, i);
1412             }));
1413           /* *INDENT-ON* */
1414         }
1415     }
1416   else
1417     {
1418       if (!m)
1419         return VNET_API_ERROR_NO_SUCH_ENTRY;
1420
1421       if (!is_lb_static_mapping (m))
1422         return VNET_API_ERROR_INVALID_VALUE;
1423
1424       /* Free external address port */
1425       if (!(sm->static_mapping_only || out2in_only))
1426         {
1427           for (i = 0; i < vec_len (sm->addresses); i++)
1428             {
1429               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1430                 {
1431                   a = sm->addresses + i;
1432                   switch (proto)
1433                     {
1434 #define _(N, j, n, s) \
1435                     case NAT_PROTOCOL_##N: \
1436                       --a->busy_##n##_port_refcounts[e_port]; \
1437                       if (e_port > 1024) \
1438                         { \
1439                           a->busy_##n##_ports--; \
1440                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1441                         } \
1442                       break;
1443                       foreach_nat_protocol
1444 #undef _
1445                     default:
1446                       nat_elog_info ("unknown protocol");
1447                       return VNET_API_ERROR_INVALID_VALUE_2;
1448                     }
1449                   break;
1450                 }
1451             }
1452         }
1453
1454       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1455       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1456         {
1457           nat_elog_err ("static_mapping_by_external key del failed");
1458           return VNET_API_ERROR_UNSPECIFIED;
1459         }
1460
1461       /* *INDENT-OFF* */
1462       pool_foreach (local, m->locals,
1463       ({
1464           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1465                             nat_fib_src_low);
1466           if (!out2in_only)
1467             {
1468 init_nat_k(&              kv, local->addr, local->port, local->fib_index, m->proto);
1469               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1470                 {
1471                   nat_elog_err ("static_mapping_by_local key del failed");
1472                   return VNET_API_ERROR_UNSPECIFIED;
1473                 }
1474             }
1475
1476           if (sm->num_workers > 1)
1477             {
1478               ip4_header_t ip = {
1479                 .src_address = local->addr,
1480               };
1481               tsm = vec_elt_at_index (sm->per_thread_data,
1482                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1483             }
1484           else
1485             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1486
1487           /* Delete sessions */
1488           pool_foreach (s, tsm->sessions, {
1489             if (!(is_lb_session (s)))
1490               continue;
1491
1492             if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1493                 s->in2out.port != local->port)
1494               continue;
1495
1496             nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1497             nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1498           });
1499       }));
1500       /* *INDENT-ON* */
1501       if (m->affinity)
1502         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1503       pool_free (m->locals);
1504       vec_free (m->tag);
1505       vec_free (m->workers);
1506
1507       pool_put (sm->static_mappings, m);
1508     }
1509
1510   return 0;
1511 }
1512
1513 int
1514 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1515                                        ip4_address_t l_addr, u16 l_port,
1516                                        nat_protocol_t proto, u32 vrf_id,
1517                                        u8 probability, u8 is_add)
1518 {
1519   snat_main_t *sm = &snat_main;
1520   snat_static_mapping_t *m = 0;
1521   clib_bihash_kv_8_8_t kv, value;
1522   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1523   snat_main_per_thread_data_t *tsm;
1524   snat_session_t *s;
1525   u32 *locals = 0;
1526   uword *bitmap = 0;
1527   int i;
1528
1529   if (!sm->endpoint_dependent)
1530     return VNET_API_ERROR_FEATURE_DISABLED;
1531
1532   init_nat_k (&kv, e_addr, e_port, 0, proto);
1533   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1534     m = pool_elt_at_index (sm->static_mappings, value.value);
1535
1536   if (!m)
1537     return VNET_API_ERROR_NO_SUCH_ENTRY;
1538
1539   if (!is_lb_static_mapping (m))
1540     return VNET_API_ERROR_INVALID_VALUE;
1541
1542   /* *INDENT-OFF* */
1543   pool_foreach (local, m->locals,
1544   ({
1545     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1546         (local->vrf_id == vrf_id))
1547       {
1548         match_local = local;
1549         break;
1550       }
1551   }));
1552   /* *INDENT-ON* */
1553
1554   if (is_add)
1555     {
1556       if (match_local)
1557         return VNET_API_ERROR_VALUE_EXIST;
1558
1559       pool_get (m->locals, local);
1560       clib_memset (local, 0, sizeof (*local));
1561       local->addr.as_u32 = l_addr.as_u32;
1562       local->port = l_port;
1563       local->probability = probability;
1564       local->vrf_id = vrf_id;
1565       local->fib_index =
1566         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1567                                            nat_fib_src_low);
1568
1569       if (!is_out2in_only_static_mapping (m))
1570         {
1571           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto,
1572                        m - sm->static_mappings);
1573           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1574             nat_elog_err ("static_mapping_by_local key add failed");
1575         }
1576     }
1577   else
1578     {
1579       if (!match_local)
1580         return VNET_API_ERROR_NO_SUCH_ENTRY;
1581
1582       if (pool_elts (m->locals) < 3)
1583         return VNET_API_ERROR_UNSPECIFIED;
1584
1585       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1586                         nat_fib_src_low);
1587
1588       if (!is_out2in_only_static_mapping (m))
1589         {
1590           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1591           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1592             nat_elog_err ("static_mapping_by_local key del failed");
1593         }
1594
1595       if (sm->num_workers > 1)
1596         {
1597           ip4_header_t ip = {
1598             .src_address = local->addr,
1599           };
1600           tsm = vec_elt_at_index (sm->per_thread_data,
1601                                   sm->worker_in2out_cb (&ip, m->fib_index,
1602                                                         0));
1603         }
1604       else
1605         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1606
1607       /* Delete sessions */
1608       /* *INDENT-OFF* */
1609       pool_foreach (s, tsm->sessions, {
1610         if (!(is_lb_session (s)))
1611           continue;
1612
1613         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1614             s->in2out.port != match_local->port)
1615           continue;
1616
1617         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1618         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1619       });
1620       /* *INDENT-ON* */
1621
1622       pool_put (m->locals, match_local);
1623     }
1624
1625   vec_free (m->workers);
1626
1627   /* *INDENT-OFF* */
1628   pool_foreach (local, m->locals,
1629   ({
1630     vec_add1 (locals, local - m->locals);
1631     if (sm->num_workers > 1)
1632       {
1633         ip4_header_t ip;
1634         ip.src_address.as_u32 = local->addr.as_u32,
1635         bitmap = clib_bitmap_set (bitmap,
1636                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1637                                   1);
1638       }
1639   }));
1640   /* *INDENT-ON* */
1641
1642   ASSERT (vec_len (locals) > 1);
1643
1644   local = pool_elt_at_index (m->locals, locals[0]);
1645   local->prefix = local->probability;
1646   for (i = 1; i < vec_len (locals); i++)
1647     {
1648       local = pool_elt_at_index (m->locals, locals[i]);
1649       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1650       local->prefix = local->probability + prev_local->prefix;
1651     }
1652
1653   /* Assign workers */
1654   if (sm->num_workers > 1)
1655     {
1656       /* *INDENT-OFF* */
1657       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1658       /* *INDENT-ON* */
1659     }
1660
1661   return 0;
1662 }
1663
1664 int
1665 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1666                   u8 twice_nat)
1667 {
1668   snat_address_t *a = 0;
1669   snat_session_t *ses;
1670   u32 *ses_to_be_removed = 0, *ses_index;
1671   snat_main_per_thread_data_t *tsm;
1672   snat_static_mapping_t *m;
1673   snat_interface_t *interface;
1674   int i;
1675   snat_address_t *addresses =
1676     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1677
1678   /* Find SNAT address */
1679   for (i = 0; i < vec_len (addresses); i++)
1680     {
1681       if (addresses[i].addr.as_u32 == addr.as_u32)
1682         {
1683           a = addresses + i;
1684           break;
1685         }
1686     }
1687   if (!a)
1688     return VNET_API_ERROR_NO_SUCH_ENTRY;
1689
1690   if (delete_sm)
1691     {
1692       ip4_address_t pool_addr = { 0 };
1693       /* *INDENT-OFF* */
1694       pool_foreach (m, sm->static_mappings,
1695       ({
1696           if (m->external_addr.as_u32 == addr.as_u32)
1697             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1698                                             m->local_port, m->external_port,
1699                                             m->vrf_id,
1700                                             is_addr_only_static_mapping(m), ~0,
1701                                             m->proto, 0 /* is_add */,
1702                                             m->twice_nat,
1703                                             is_out2in_only_static_mapping(m),
1704                                             m->tag,
1705                                             is_identity_static_mapping(m),
1706                                             pool_addr, 0);
1707       }));
1708       /* *INDENT-ON* */
1709     }
1710   else
1711     {
1712       /* Check if address is used in some static mapping */
1713       if (is_snat_address_used_in_static_mapping (sm, addr))
1714         {
1715           nat_elog_notice ("address used in static mapping");
1716           return VNET_API_ERROR_UNSPECIFIED;
1717         }
1718     }
1719
1720   if (a->fib_index != ~0)
1721     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1722
1723   /* Delete sessions using address */
1724   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1725     {
1726       /* *INDENT-OFF* */
1727       vec_foreach (tsm, sm->per_thread_data)
1728         {
1729           pool_foreach (ses, tsm->sessions, ({
1730             if (ses->out2in.addr.as_u32 == addr.as_u32)
1731               {
1732                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1733                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1734               }
1735           }));
1736
1737           if (sm->endpoint_dependent){
1738               vec_foreach (ses_index, ses_to_be_removed)
1739                 {
1740                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1741                   nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1742                 }
1743           }else{
1744               vec_foreach (ses_index, ses_to_be_removed)
1745                 {
1746                   ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1747                   nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1748                 }
1749           }
1750
1751           vec_free (ses_to_be_removed);
1752         }
1753       /* *INDENT-ON* */
1754     }
1755
1756 #define _(N, i, n, s) \
1757   vec_free (a->busy_##n##_ports_per_thread);
1758   foreach_nat_protocol
1759 #undef _
1760     if (twice_nat)
1761     {
1762       vec_del1 (sm->twice_nat_addresses, i);
1763       return 0;
1764     }
1765   else
1766     vec_del1 (sm->addresses, i);
1767
1768   /* Delete external address from FIB */
1769   /* *INDENT-OFF* */
1770   pool_foreach (interface, sm->interfaces,
1771   ({
1772     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1773       continue;
1774
1775     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1776     break;
1777   }));
1778   pool_foreach (interface, sm->output_feature_interfaces,
1779   ({
1780     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1781       continue;
1782
1783     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1784     break;
1785   }));
1786   /* *INDENT-ON* */
1787
1788   return 0;
1789 }
1790
1791 static void
1792 nat_validate_counters (snat_main_t * sm, u32 sw_if_index)
1793 {
1794 #define _(x)                                                                  \
1795   vlib_validate_simple_counter (&sm->counters.fastpath.in2out.x,              \
1796                                 sw_if_index);                                 \
1797   vlib_zero_simple_counter (&sm->counters.fastpath.in2out.x, sw_if_index);    \
1798   vlib_validate_simple_counter (&sm->counters.fastpath.out2in.x,              \
1799                                 sw_if_index);                                 \
1800   vlib_zero_simple_counter (&sm->counters.fastpath.out2in.x, sw_if_index);    \
1801   vlib_validate_simple_counter (&sm->counters.slowpath.in2out.x,              \
1802                                 sw_if_index);                                 \
1803   vlib_zero_simple_counter (&sm->counters.slowpath.in2out.x, sw_if_index);    \
1804   vlib_validate_simple_counter (&sm->counters.slowpath.out2in.x,              \
1805                                 sw_if_index);                                 \
1806   vlib_zero_simple_counter (&sm->counters.slowpath.out2in.x, sw_if_index);    \
1807   vlib_validate_simple_counter (&sm->counters.fastpath.in2out_ed.x,           \
1808                                 sw_if_index);                                 \
1809   vlib_zero_simple_counter (&sm->counters.fastpath.in2out_ed.x, sw_if_index); \
1810   vlib_validate_simple_counter (&sm->counters.fastpath.out2in_ed.x,           \
1811                                 sw_if_index);                                 \
1812   vlib_zero_simple_counter (&sm->counters.fastpath.out2in_ed.x, sw_if_index); \
1813   vlib_validate_simple_counter (&sm->counters.slowpath.in2out_ed.x,           \
1814                                 sw_if_index);                                 \
1815   vlib_zero_simple_counter (&sm->counters.slowpath.in2out_ed.x, sw_if_index); \
1816   vlib_validate_simple_counter (&sm->counters.slowpath.out2in_ed.x,           \
1817                                 sw_if_index);                                 \
1818   vlib_zero_simple_counter (&sm->counters.slowpath.out2in_ed.x, sw_if_index);
1819   foreach_nat_counter;
1820 #undef _
1821   vlib_validate_simple_counter (&sm->counters.hairpinning, sw_if_index);
1822   vlib_zero_simple_counter (&sm->counters.hairpinning, sw_if_index);
1823 }
1824
1825 void
1826 expire_per_vrf_sessions (u32 fib_index)
1827 {
1828   per_vrf_sessions_t *per_vrf_sessions;
1829   snat_main_per_thread_data_t *tsm;
1830   snat_main_t *sm = &snat_main;
1831
1832   /* *INDENT-OFF* */
1833   vec_foreach (tsm, sm->per_thread_data)
1834     {
1835       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1836         {
1837           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1838               (per_vrf_sessions->tx_fib_index == fib_index))
1839             {
1840               per_vrf_sessions->expired = 1;
1841             }
1842         }
1843     }
1844   /* *INDENT-ON* */
1845 }
1846
1847 void
1848 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1849 {
1850   snat_main_t *sm = &snat_main;
1851   nat_fib_t *fib;
1852
1853   // we don't care if it is outside/inside fib
1854   // we just care about their ref_count
1855   // if it reaches 0 sessions should expire
1856   // because the fib isn't valid for NAT anymore
1857
1858   vec_foreach (fib, sm->fibs)
1859   {
1860     if (fib->fib_index == fib_index)
1861       {
1862         if (is_del)
1863           {
1864             fib->ref_count--;
1865             if (!fib->ref_count)
1866               {
1867                 vec_del1 (sm->fibs, fib - sm->fibs);
1868                 expire_per_vrf_sessions (fib_index);
1869               }
1870             return;
1871           }
1872         else
1873           fib->ref_count++;
1874       }
1875   }
1876   if (!is_del)
1877     {
1878       vec_add2 (sm->fibs, fib, 1);
1879       fib->ref_count = 1;
1880       fib->fib_index = fib_index;
1881     }
1882 }
1883
1884 int
1885 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1886 {
1887   snat_main_t *sm = &snat_main;
1888   snat_interface_t *i;
1889   const char *feature_name, *del_feature_name;
1890   snat_address_t *ap;
1891   snat_static_mapping_t *m;
1892   nat_outside_fib_t *outside_fib;
1893   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1894                                                        sw_if_index);
1895
1896   if (sm->out2in_dpo && !is_inside)
1897     return VNET_API_ERROR_UNSUPPORTED;
1898
1899   /* *INDENT-OFF* */
1900   pool_foreach (i, sm->output_feature_interfaces,
1901   ({
1902     if (i->sw_if_index == sw_if_index)
1903       return VNET_API_ERROR_VALUE_EXIST;
1904   }));
1905   /* *INDENT-ON* */
1906
1907   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1908     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1909   else
1910     {
1911       if (sm->num_workers > 1)
1912         feature_name =
1913           is_inside ? "nat44-in2out-worker-handoff" :
1914           "nat44-out2in-worker-handoff";
1915       else if (sm->endpoint_dependent)
1916         {
1917           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1918         }
1919       else
1920         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1921     }
1922
1923   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
1924     sm->fq_in2out_index =
1925       vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
1926
1927   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1928     sm->fq_out2in_index =
1929       vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
1930
1931   if (sm->endpoint_dependent)
1932     update_per_vrf_sessions_vec (fib_index, is_del);
1933
1934   if (!is_inside)
1935     {
1936       /* *INDENT-OFF* */
1937       vec_foreach (outside_fib, sm->outside_fibs)
1938         {
1939           if (outside_fib->fib_index == fib_index)
1940             {
1941               if (is_del)
1942                 {
1943                   outside_fib->refcount--;
1944                   if (!outside_fib->refcount)
1945                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1946                 }
1947               else
1948                 outside_fib->refcount++;
1949               goto feature_set;
1950             }
1951         }
1952       /* *INDENT-ON* */
1953       if (!is_del)
1954         {
1955           vec_add2 (sm->outside_fibs, outside_fib, 1);
1956           outside_fib->refcount = 1;
1957           outside_fib->fib_index = fib_index;
1958         }
1959     }
1960
1961 feature_set:
1962   /* *INDENT-OFF* */
1963   pool_foreach (i, sm->interfaces,
1964   ({
1965     if (i->sw_if_index == sw_if_index)
1966       {
1967         if (is_del)
1968           {
1969             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1970               {
1971                 if (is_inside)
1972                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1973                 else
1974                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1975
1976                 if (sm->num_workers > 1)
1977                   {
1978                     del_feature_name = "nat44-handoff-classify";
1979                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1980                                                  "nat44-out2in-worker-handoff";
1981                   }
1982                 else if (sm->endpoint_dependent)
1983                   {
1984                     del_feature_name = "nat44-ed-classify";
1985                     feature_name = !is_inside ?  "nat-pre-in2out" :
1986                                                  "nat-pre-out2in";
1987                   }
1988                 else
1989                   {
1990                     del_feature_name = "nat44-classify";
1991                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1992                   }
1993
1994                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1995                 if (rv)
1996                   return rv;
1997                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1998                                              sw_if_index, 0, 0, 0);
1999                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2000                                              sw_if_index, 1, 0, 0);
2001                 if (!is_inside)
2002                   {
2003                     if (sm->endpoint_dependent)
2004                       vnet_feature_enable_disable ("ip4-local",
2005                                                    "nat44-ed-hairpinning",
2006                                                    sw_if_index, 1, 0, 0);
2007                     else
2008                       vnet_feature_enable_disable ("ip4-local",
2009                                                    "nat44-hairpinning",
2010                                                    sw_if_index, 1, 0, 0);
2011                   }
2012               }
2013             else
2014               {
2015                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2016                 if (rv)
2017                   return rv;
2018                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
2019                                              sw_if_index, 0, 0, 0);
2020                 pool_put (sm->interfaces, i);
2021                 if (is_inside)
2022                   {
2023                     if (sm->endpoint_dependent)
2024                       vnet_feature_enable_disable ("ip4-local",
2025                                                    "nat44-ed-hairpinning",
2026                                                    sw_if_index, 0, 0, 0);
2027                     else
2028                       vnet_feature_enable_disable ("ip4-local",
2029                                                    "nat44-hairpinning",
2030                                                    sw_if_index, 0, 0, 0);
2031                   }
2032               }
2033           }
2034         else
2035           {
2036             if ((nat_interface_is_inside(i) && is_inside) ||
2037                 (nat_interface_is_outside(i) && !is_inside))
2038               return 0;
2039
2040             if (sm->num_workers > 1)
2041               {
2042                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
2043                                                  "nat44-out2in-worker-handoff";
2044                 feature_name = "nat44-handoff-classify";
2045               }
2046             else if (sm->endpoint_dependent)
2047               {
2048                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
2049                                                  "nat-pre-out2in";
2050
2051                 feature_name = "nat44-ed-classify";
2052               }
2053             else
2054               {
2055                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
2056                 feature_name = "nat44-classify";
2057               }
2058
2059             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2060             if (rv)
2061               return rv;
2062             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
2063                                          sw_if_index, 0, 0, 0);
2064             vnet_feature_enable_disable ("ip4-unicast", feature_name,
2065                                          sw_if_index, 1, 0, 0);
2066             if (!is_inside)
2067               {
2068                 if (sm->endpoint_dependent)
2069                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2070                                                sw_if_index, 0, 0, 0);
2071                 else
2072                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2073                                                sw_if_index, 0, 0, 0);
2074               }
2075             goto set_flags;
2076           }
2077
2078         goto fib;
2079       }
2080   }));
2081   /* *INDENT-ON* */
2082
2083   if (is_del)
2084     return VNET_API_ERROR_NO_SUCH_ENTRY;
2085
2086   pool_get (sm->interfaces, i);
2087   i->sw_if_index = sw_if_index;
2088   i->flags = 0;
2089   nat_validate_counters (sm, sw_if_index);
2090
2091   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
2092                                0);
2093
2094   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
2095   if (rv)
2096     return rv;
2097
2098   if (is_inside && !sm->out2in_dpo)
2099     {
2100       if (sm->endpoint_dependent)
2101         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
2102                                      sw_if_index, 1, 0, 0);
2103       else
2104         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
2105                                      sw_if_index, 1, 0, 0);
2106     }
2107
2108 set_flags:
2109   if (is_inside)
2110     {
2111       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2112       return 0;
2113     }
2114   else
2115     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2116
2117   /* Add/delete external addresses to FIB */
2118 fib:
2119   /* *INDENT-OFF* */
2120   vec_foreach (ap, sm->addresses)
2121     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2122
2123   pool_foreach (m, sm->static_mappings,
2124   ({
2125     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2126       continue;
2127
2128     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2129   }));
2130   /* *INDENT-ON* */
2131
2132   return 0;
2133 }
2134
2135 int
2136 snat_interface_add_del_output_feature (u32 sw_if_index,
2137                                        u8 is_inside, int is_del)
2138 {
2139   snat_main_t *sm = &snat_main;
2140   snat_interface_t *i;
2141   snat_address_t *ap;
2142   snat_static_mapping_t *m;
2143   nat_outside_fib_t *outside_fib;
2144   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2145                                                        sw_if_index);
2146
2147   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
2148     return VNET_API_ERROR_UNSUPPORTED;
2149
2150   /* *INDENT-OFF* */
2151   pool_foreach (i, sm->interfaces,
2152   ({
2153     if (i->sw_if_index == sw_if_index)
2154       return VNET_API_ERROR_VALUE_EXIST;
2155   }));
2156   /* *INDENT-ON* */
2157
2158   if (sm->endpoint_dependent)
2159     update_per_vrf_sessions_vec (fib_index, is_del);
2160
2161   if (!is_inside)
2162     {
2163       /* *INDENT-OFF* */
2164       vec_foreach (outside_fib, sm->outside_fibs)
2165         {
2166           if (outside_fib->fib_index == fib_index)
2167             {
2168               if (is_del)
2169                 {
2170                   outside_fib->refcount--;
2171                   if (!outside_fib->refcount)
2172                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2173                 }
2174               else
2175                 outside_fib->refcount++;
2176               goto feature_set;
2177             }
2178         }
2179       /* *INDENT-ON* */
2180       if (!is_del)
2181         {
2182           vec_add2 (sm->outside_fibs, outside_fib, 1);
2183           outside_fib->refcount = 1;
2184           outside_fib->fib_index = fib_index;
2185         }
2186     }
2187
2188 feature_set:
2189   if (is_inside)
2190     {
2191       if (sm->endpoint_dependent)
2192         {
2193           int rv =
2194             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2195           if (rv)
2196             return rv;
2197           rv =
2198             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2199                                                             !is_del);
2200           if (rv)
2201             return rv;
2202           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2203                                        sw_if_index, !is_del, 0, 0);
2204           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2205                                        sw_if_index, !is_del, 0, 0);
2206         }
2207       else
2208         {
2209           int rv =
2210             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2211           if (rv)
2212             return rv;
2213           rv =
2214             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2215                                                             !is_del);
2216           if (rv)
2217             return rv;
2218           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2219                                        sw_if_index, !is_del, 0, 0);
2220           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2221                                        sw_if_index, !is_del, 0, 0);
2222         }
2223       goto fq;
2224     }
2225
2226   if (sm->num_workers > 1)
2227     {
2228       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2229       if (rv)
2230         return rv;
2231       rv =
2232         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2233       if (rv)
2234         return rv;
2235       vnet_feature_enable_disable ("ip4-unicast",
2236                                    "nat44-out2in-worker-handoff",
2237                                    sw_if_index, !is_del, 0, 0);
2238       vnet_feature_enable_disable ("ip4-output",
2239                                    "nat44-in2out-output-worker-handoff",
2240                                    sw_if_index, !is_del, 0, 0);
2241     }
2242   else
2243     {
2244       if (sm->endpoint_dependent)
2245         {
2246           int rv =
2247             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2248           if (rv)
2249             return rv;
2250           rv =
2251             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2252                                                             !is_del);
2253           if (rv)
2254             return rv;
2255           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2256                                        sw_if_index, !is_del, 0, 0);
2257           vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2258                                        sw_if_index, !is_del, 0, 0);
2259         }
2260       else
2261         {
2262           int rv =
2263             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2264           if (rv)
2265             return rv;
2266           rv =
2267             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2268                                                             !is_del);
2269           if (rv)
2270             return rv;
2271           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2272                                        sw_if_index, !is_del, 0, 0);
2273           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2274                                        sw_if_index, !is_del, 0, 0);
2275         }
2276     }
2277
2278 fq:
2279   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2280     sm->fq_in2out_output_index =
2281       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
2282
2283   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2284     sm->fq_out2in_index =
2285       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
2286
2287   /* *INDENT-OFF* */
2288   pool_foreach (i, sm->output_feature_interfaces,
2289   ({
2290     if (i->sw_if_index == sw_if_index)
2291       {
2292         if (is_del)
2293           pool_put (sm->output_feature_interfaces, i);
2294         else
2295           return VNET_API_ERROR_VALUE_EXIST;
2296
2297         goto fib;
2298       }
2299   }));
2300   /* *INDENT-ON* */
2301
2302   if (is_del)
2303     return VNET_API_ERROR_NO_SUCH_ENTRY;
2304
2305   pool_get (sm->output_feature_interfaces, i);
2306   i->sw_if_index = sw_if_index;
2307   i->flags = 0;
2308   nat_validate_counters (sm, sw_if_index);
2309   if (is_inside)
2310     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2311   else
2312     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2313
2314   /* Add/delete external addresses to FIB */
2315 fib:
2316   if (is_inside)
2317     return 0;
2318
2319   /* *INDENT-OFF* */
2320   vec_foreach (ap, sm->addresses)
2321     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2322
2323   pool_foreach (m, sm->static_mappings,
2324   ({
2325     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2326       continue;
2327
2328     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2329   }));
2330   /* *INDENT-ON* */
2331
2332   return 0;
2333 }
2334
2335 int
2336 snat_set_workers (uword * bitmap)
2337 {
2338   snat_main_t *sm = &snat_main;
2339   int i, j = 0;
2340
2341   if (sm->num_workers < 2)
2342     return VNET_API_ERROR_FEATURE_DISABLED;
2343
2344   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2345     return VNET_API_ERROR_INVALID_WORKER;
2346
2347   vec_free (sm->workers);
2348   /* *INDENT-OFF* */
2349   clib_bitmap_foreach (i, bitmap,
2350     ({
2351       vec_add1(sm->workers, i);
2352       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2353       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2354       j++;
2355     }));
2356   /* *INDENT-ON* */
2357
2358   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2359
2360   return 0;
2361 }
2362
2363 static void
2364 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2365                          u32 old_fib_index)
2366 {
2367   snat_main_t *sm = &snat_main;
2368   nat_outside_fib_t *outside_fib;
2369   snat_interface_t *i;
2370   u8 is_add = 1;
2371   u8 match = 0;
2372
2373   if (new_fib_index == old_fib_index)
2374     return;
2375
2376   if (!vec_len (sm->outside_fibs))
2377     return;
2378
2379   /* *INDENT-OFF* */
2380   pool_foreach (i, sm->interfaces,
2381     ({
2382       if (i->sw_if_index == sw_if_index)
2383         {
2384           if (!(nat_interface_is_outside (i)))
2385             return;
2386           match = 1;
2387         }
2388     }));
2389
2390   pool_foreach (i, sm->output_feature_interfaces,
2391     ({
2392       if (i->sw_if_index == sw_if_index)
2393         {
2394           if (!(nat_interface_is_outside (i)))
2395             return;
2396           match = 1;
2397         }
2398     }));
2399   /* *INDENT-ON* */
2400
2401   if (!match)
2402     return;
2403
2404   vec_foreach (outside_fib, sm->outside_fibs)
2405   {
2406     if (outside_fib->fib_index == old_fib_index)
2407       {
2408         outside_fib->refcount--;
2409         if (!outside_fib->refcount)
2410           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2411         break;
2412       }
2413   }
2414
2415   vec_foreach (outside_fib, sm->outside_fibs)
2416   {
2417     if (outside_fib->fib_index == new_fib_index)
2418       {
2419         outside_fib->refcount++;
2420         is_add = 0;
2421         break;
2422       }
2423   }
2424
2425   if (is_add)
2426     {
2427       vec_add2 (sm->outside_fibs, outside_fib, 1);
2428       outside_fib->refcount = 1;
2429       outside_fib->fib_index = new_fib_index;
2430     }
2431 }
2432
2433 static void
2434 snat_ip4_table_bind (ip4_main_t * im,
2435                      uword opaque,
2436                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2437 {
2438   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2439 }
2440
2441 static void
2442 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2443                                        uword opaque,
2444                                        u32 sw_if_index,
2445                                        ip4_address_t * address,
2446                                        u32 address_length,
2447                                        u32 if_address_index, u32 is_delete);
2448
2449 static void
2450 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2451                                  uword opaque,
2452                                  u32 sw_if_index,
2453                                  ip4_address_t * address,
2454                                  u32 address_length,
2455                                  u32 if_address_index, u32 is_delete);
2456
2457 static int
2458 nat_alloc_addr_and_port_default (snat_address_t * addresses, u32 fib_index,
2459                                  u32 thread_index, nat_protocol_t proto,
2460                                  ip4_address_t * addr, u16 * port,
2461                                  u16 port_per_thread, u32 snat_thread_index);
2462
2463 void
2464 test_key_calc_split ()
2465 {
2466   ip4_address_t l_addr;
2467   l_addr.as_u8[0] = 1;
2468   l_addr.as_u8[1] = 1;
2469   l_addr.as_u8[2] = 1;
2470   l_addr.as_u8[3] = 1;
2471   ip4_address_t r_addr;
2472   r_addr.as_u8[0] = 2;
2473   r_addr.as_u8[1] = 2;
2474   r_addr.as_u8[2] = 2;
2475   r_addr.as_u8[3] = 2;
2476   u16 l_port = 40001;
2477   u16 r_port = 40301;
2478   u8 proto = 9;
2479   u32 fib_index = 9000001;
2480   u32 thread_index = 3000000001;
2481   u32 session_index = 3000000221;
2482   clib_bihash_kv_16_8_t kv;
2483   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2484               thread_index, session_index);
2485   ip4_address_t l_addr2;
2486   ip4_address_t r_addr2;
2487   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2488   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2489   u16 l_port2 = 0;
2490   u16 r_port2 = 0;
2491   u8 proto2 = 0;
2492   u32 fib_index2 = 0;
2493   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2494                &r_port2);
2495   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2496   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2497   ASSERT (l_port == l_port2);
2498   ASSERT (r_port == r_port2);
2499   ASSERT (proto == proto2);
2500   ASSERT (fib_index == fib_index2);
2501   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2502   ASSERT (session_index == ed_value_get_session_index (&kv));
2503
2504   fib_index = 7001;
2505   proto = 5;
2506   nat_protocol_t proto3 = ~0;
2507   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2508   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2509   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2510   ASSERT (l_port == l_port2);
2511   ASSERT (proto == proto3);
2512   ASSERT (fib_index == fib_index2);
2513 }
2514
2515 static clib_error_t *
2516 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2517 {
2518   snat_main_t *sm = &snat_main;
2519   u32 fib_index;
2520
2521   if (sm->endpoint_dependent)
2522     {
2523       // TODO: consider removing all NAT interfaces
2524
2525       if (!is_add)
2526         {
2527           fib_index = ip4_fib_index_from_table_id (table_id);
2528           if (fib_index != ~0)
2529             expire_per_vrf_sessions (fib_index);
2530         }
2531     }
2532   return 0;
2533 }
2534
2535 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2536
2537
2538 static clib_error_t *
2539 snat_init (vlib_main_t * vm)
2540 {
2541   snat_main_t *sm = &snat_main;
2542   clib_error_t *error = 0;
2543   ip4_main_t *im = &ip4_main;
2544   ip_lookup_main_t *lm = &im->lookup_main;
2545   uword *p;
2546   vlib_thread_registration_t *tr;
2547   vlib_thread_main_t *tm = vlib_get_thread_main ();
2548   uword *bitmap = 0;
2549   u32 i;
2550   ip4_add_del_interface_address_callback_t cb4;
2551   vlib_node_t *node;
2552
2553   sm->vnet_main = vnet_get_main ();
2554   sm->ip4_main = im;
2555   sm->ip4_lookup_main = lm;
2556   sm->api_main = vlibapi_get_main ();
2557   sm->first_worker_index = 0;
2558   sm->num_workers = 0;
2559   sm->workers = 0;
2560   sm->port_per_thread = 0xffff - 1024;
2561   sm->fq_in2out_index = ~0;
2562   sm->fq_in2out_output_index = ~0;
2563   sm->fq_out2in_index = ~0;
2564
2565   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2566   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2567   sm->forwarding_enabled = 0;
2568   sm->log_class = vlib_log_register_class ("nat", 0);
2569   sm->log_level = SNAT_LOG_ERROR;
2570   sm->mss_clamping = 0;
2571
2572   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2573   sm->error_node_index = node->index;
2574
2575   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2576   sm->pre_in2out_node_index = node->index;
2577   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2578   sm->pre_out2in_node_index = node->index;
2579
2580   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2581   sm->pre_in2out_node_index = node->index;
2582
2583   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2584   sm->pre_out2in_node_index = node->index;
2585
2586   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2587   sm->in2out_node_index = node->index;
2588   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2589   sm->in2out_output_node_index = node->index;
2590   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2591   sm->in2out_fast_node_index = node->index;
2592   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2593   sm->in2out_slowpath_node_index = node->index;
2594   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2595   sm->in2out_slowpath_output_node_index = node->index;
2596
2597   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2598   sm->ed_in2out_node_index = node->index;
2599   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2600   sm->ed_in2out_slowpath_node_index = node->index;
2601
2602   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2603   sm->out2in_node_index = node->index;
2604   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2605   sm->out2in_fast_node_index = node->index;
2606
2607   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2608   sm->ed_out2in_node_index = node->index;
2609   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2610   sm->ed_out2in_slowpath_node_index = node->index;
2611
2612   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2613   sm->hairpinning_node_index = node->index;
2614   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2615   sm->hairpin_dst_node_index = node->index;
2616   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2617   sm->hairpin_src_node_index = node->index;
2618   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2619   sm->ed_hairpinning_node_index = node->index;
2620   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2621   sm->ed_hairpin_dst_node_index = node->index;
2622   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2623   sm->ed_hairpin_src_node_index = node->index;
2624
2625   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2626   if (p)
2627     {
2628       tr = (vlib_thread_registration_t *) p[0];
2629       if (tr)
2630         {
2631           sm->num_workers = tr->count;
2632           sm->first_worker_index = tr->first_index;
2633         }
2634     }
2635
2636   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2637
2638   /* Use all available workers by default */
2639   if (sm->num_workers > 1)
2640     {
2641       for (i = 0; i < sm->num_workers; i++)
2642         bitmap = clib_bitmap_set (bitmap, i, 1);
2643       snat_set_workers (bitmap);
2644       clib_bitmap_free (bitmap);
2645     }
2646   else
2647     {
2648       sm->per_thread_data[0].snat_thread_index = 0;
2649     }
2650
2651   error = snat_api_init (vm, sm);
2652   if (error)
2653     return error;
2654
2655   /* Set up the interface address add/del callback */
2656   cb4.function = snat_ip4_add_del_interface_address_cb;
2657   cb4.function_opaque = 0;
2658
2659   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2660
2661   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2662   cb4.function_opaque = 0;
2663
2664   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2665
2666   nat_dpo_module_init ();
2667
2668   /* Init counters */
2669   sm->total_users.name = "total-users";
2670   sm->total_users.stat_segment_name = "/nat44/total-users";
2671   vlib_validate_simple_counter (&sm->total_users, 0);
2672   vlib_zero_simple_counter (&sm->total_users, 0);
2673   sm->total_sessions.name = "total-sessions";
2674   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2675   vlib_validate_simple_counter (&sm->total_sessions, 0);
2676   vlib_zero_simple_counter (&sm->total_sessions, 0);
2677   sm->user_limit_reached.name = "user-limit-reached";
2678   sm->user_limit_reached.stat_segment_name = "/nat44/user-limit-reached";
2679   vlib_validate_simple_counter (&sm->user_limit_reached, 0);
2680   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
2681
2682 #define _(x)                                            \
2683   sm->counters.fastpath.in2out.x.name = #x;             \
2684   sm->counters.fastpath.in2out.x.stat_segment_name =    \
2685       "/nat44/in2out/fastpath/" #x;                     \
2686   sm->counters.slowpath.in2out.x.name = #x;             \
2687   sm->counters.slowpath.in2out.x.stat_segment_name =    \
2688       "/nat44/in2out/slowpath/" #x;                     \
2689   sm->counters.fastpath.out2in.x.name = #x;             \
2690   sm->counters.fastpath.out2in.x.stat_segment_name =    \
2691       "/nat44/out2in/fastpath/" #x;                     \
2692   sm->counters.slowpath.out2in.x.name = #x;             \
2693   sm->counters.slowpath.out2in.x.stat_segment_name =    \
2694       "/nat44/out2in/slowpath/" #x;                     \
2695   sm->counters.fastpath.in2out_ed.x.name = #x;          \
2696   sm->counters.fastpath.in2out_ed.x.stat_segment_name = \
2697       "/nat44/ed/in2out/fastpath/" #x;                  \
2698   sm->counters.slowpath.in2out_ed.x.name = #x;          \
2699   sm->counters.slowpath.in2out_ed.x.stat_segment_name = \
2700       "/nat44/ed/in2out/slowpath/" #x;                  \
2701   sm->counters.fastpath.out2in_ed.x.name = #x;          \
2702   sm->counters.fastpath.out2in_ed.x.stat_segment_name = \
2703       "/nat44/ed/out2in/fastpath/" #x;                  \
2704   sm->counters.slowpath.out2in_ed.x.name = #x;          \
2705   sm->counters.slowpath.out2in_ed.x.stat_segment_name = \
2706       "/nat44/ed/out2in/slowpath/" #x;
2707   foreach_nat_counter;
2708 #undef _
2709   sm->counters.hairpinning.name = "hairpinning";
2710   sm->counters.hairpinning.stat_segment_name = "/nat44/hairpinning";
2711
2712   /* Init IPFIX logging */
2713   nat_ipfix_logging_init (vm);
2714
2715   ip4_table_bind_callback_t cbt4 = {
2716     .function = snat_ip4_table_bind,
2717   };
2718   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2719
2720   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2721                                         FIB_SOURCE_PRIORITY_HI,
2722                                         FIB_SOURCE_BH_SIMPLE);
2723   nat_fib_src_low = fib_source_allocate ("nat-low",
2724                                          FIB_SOURCE_PRIORITY_LOW,
2725                                          FIB_SOURCE_BH_SIMPLE);
2726
2727   test_key_calc_split ();
2728   return error;
2729 }
2730
2731 VLIB_INIT_FUNCTION (snat_init);
2732
2733 void
2734 snat_free_outside_address_and_port (snat_address_t * addresses,
2735                                     u32 thread_index,
2736                                     ip4_address_t * addr,
2737                                     u16 port, nat_protocol_t protocol)
2738 {
2739   snat_address_t *a;
2740   u32 address_index;
2741   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2742
2743   for (address_index = 0; address_index < vec_len (addresses);
2744        address_index++)
2745     {
2746       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2747         break;
2748     }
2749
2750   ASSERT (address_index < vec_len (addresses));
2751
2752   a = addresses + address_index;
2753
2754   switch (protocol)
2755     {
2756 #define _(N, i, n, s) \
2757     case NAT_PROTOCOL_##N: \
2758       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2759       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2760       a->busy_##n##_ports--; \
2761       a->busy_##n##_ports_per_thread[thread_index]--; \
2762       break;
2763       foreach_nat_protocol
2764 #undef _
2765     default:
2766       nat_elog_info ("unknown protocol");
2767       return;
2768     }
2769 }
2770
2771 static int
2772 nat_set_outside_address_and_port (snat_address_t * addresses,
2773                                   u32 thread_index, ip4_address_t addr,
2774                                   u16 port, nat_protocol_t protocol)
2775 {
2776   snat_address_t *a = 0;
2777   u32 address_index;
2778   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2779
2780   for (address_index = 0; address_index < vec_len (addresses);
2781        address_index++)
2782     {
2783       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2784         continue;
2785
2786       a = addresses + address_index;
2787       switch (protocol)
2788         {
2789 #define _(N, j, n, s) \
2790         case NAT_PROTOCOL_##N: \
2791           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2792             return VNET_API_ERROR_INSTANCE_IN_USE; \
2793           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2794           a->busy_##n##_ports_per_thread[thread_index]++; \
2795           a->busy_##n##_ports++; \
2796           return 0;
2797           foreach_nat_protocol
2798 #undef _
2799         default:
2800           nat_elog_info ("unknown protocol");
2801           return 1;
2802         }
2803     }
2804
2805   return VNET_API_ERROR_NO_SUCH_ENTRY;
2806 }
2807
2808 int
2809 snat_static_mapping_match (snat_main_t * sm,
2810                            ip4_address_t match_addr,
2811                            u16 match_port,
2812                            u32 match_fib_index,
2813                            nat_protocol_t match_protocol,
2814                            ip4_address_t * mapping_addr,
2815                            u16 * mapping_port,
2816                            u32 * mapping_fib_index,
2817                            u8 by_external,
2818                            u8 * is_addr_only,
2819                            twice_nat_type_t * twice_nat,
2820                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2821                            u8 * is_identity_nat, snat_static_mapping_t ** out)
2822 {
2823   clib_bihash_kv_8_8_t kv, value;
2824   clib_bihash_8_8_t *mapping_hash;
2825   snat_static_mapping_t *m;
2826   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2827   nat44_lb_addr_port_t *local;
2828   u8 backend_index;
2829
2830   if (!by_external)
2831     {
2832       mapping_hash = &sm->static_mapping_by_local;
2833       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2834                   match_protocol);
2835       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2836         {
2837           /* Try address only mapping */
2838           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2839           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2840             return 1;
2841         }
2842     }
2843   else
2844     {
2845       mapping_hash = &sm->static_mapping_by_external;
2846       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2847       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2848         {
2849           /* Try address only mapping */
2850           init_nat_k (&kv, match_addr, 0, 0, 0);
2851           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2852             return 1;
2853         }
2854     }
2855
2856   m = pool_elt_at_index (sm->static_mappings, value.value);
2857
2858   if (by_external)
2859     {
2860       if (is_lb_static_mapping (m))
2861         {
2862           if (PREDICT_FALSE (lb != 0))
2863             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2864           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2865                                                           match_addr,
2866                                                           match_protocol,
2867                                                           match_port,
2868                                                           &backend_index))
2869             {
2870               local = pool_elt_at_index (m->locals, backend_index);
2871               *mapping_addr = local->addr;
2872               *mapping_port = local->port;
2873               *mapping_fib_index = local->fib_index;
2874               goto end;
2875             }
2876           // pick locals matching this worker
2877           if (PREDICT_FALSE (sm->num_workers > 1))
2878             {
2879               u32 thread_index = vlib_get_thread_index ();
2880               /* *INDENT-OFF* */
2881               pool_foreach_index (i, m->locals,
2882               ({
2883                 local = pool_elt_at_index (m->locals, i);
2884
2885                 ip4_header_t ip = {
2886                   .src_address = local->addr,
2887                 };
2888
2889                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2890                     thread_index)
2891                   {
2892                     vec_add1 (tmp, i);
2893                   }
2894               }));
2895               /* *INDENT-ON* */
2896               ASSERT (vec_len (tmp) != 0);
2897             }
2898           else
2899             {
2900               /* *INDENT-OFF* */
2901               pool_foreach_index (i, m->locals,
2902               ({
2903                 vec_add1 (tmp, i);
2904               }));
2905               /* *INDENT-ON* */
2906             }
2907           hi = vec_len (tmp) - 1;
2908           local = pool_elt_at_index (m->locals, tmp[hi]);
2909           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2910           while (lo < hi)
2911             {
2912               mid = ((hi - lo) >> 1) + lo;
2913               local = pool_elt_at_index (m->locals, tmp[mid]);
2914               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2915             }
2916           local = pool_elt_at_index (m->locals, tmp[lo]);
2917           if (!(local->prefix >= rand))
2918             return 1;
2919           *mapping_addr = local->addr;
2920           *mapping_port = local->port;
2921           *mapping_fib_index = local->fib_index;
2922           if (m->affinity)
2923             {
2924               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2925                                                 match_protocol, match_port,
2926                                                 tmp[lo], m->affinity,
2927                                                 m->affinity_per_service_list_head_index))
2928                 nat_elog_info ("create affinity record failed");
2929             }
2930           vec_free (tmp);
2931         }
2932       else
2933         {
2934           if (PREDICT_FALSE (lb != 0))
2935             *lb = NO_LB_NAT;
2936           *mapping_fib_index = m->fib_index;
2937           *mapping_addr = m->local_addr;
2938           /* Address only mapping doesn't change port */
2939           *mapping_port = is_addr_only_static_mapping (m) ? match_port
2940             : m->local_port;
2941         }
2942     }
2943   else
2944     {
2945       *mapping_addr = m->external_addr;
2946       /* Address only mapping doesn't change port */
2947       *mapping_port = is_addr_only_static_mapping (m) ? match_port
2948         : m->external_port;
2949       *mapping_fib_index = sm->outside_fib_index;
2950     }
2951
2952 end:
2953   if (PREDICT_FALSE (is_addr_only != 0))
2954     *is_addr_only = is_addr_only_static_mapping (m);
2955
2956   if (PREDICT_FALSE (twice_nat != 0))
2957     *twice_nat = m->twice_nat;
2958
2959   if (PREDICT_FALSE (is_identity_nat != 0))
2960     *is_identity_nat = is_identity_static_mapping (m);
2961
2962   if (out != 0)
2963     *out = m;
2964
2965   return 0;
2966 }
2967
2968 int
2969 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2970                                      u32 fib_index,
2971                                      u32 thread_index,
2972                                      nat_protocol_t proto,
2973                                      ip4_address_t * addr,
2974                                      u16 * port,
2975                                      u16 port_per_thread,
2976                                      u32 snat_thread_index)
2977 {
2978   snat_main_t *sm = &snat_main;
2979
2980   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, proto,
2981                                   addr, port, port_per_thread,
2982                                   snat_thread_index);
2983 }
2984
2985 static int
2986 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2987                                  u32 fib_index,
2988                                  u32 thread_index,
2989                                  nat_protocol_t proto,
2990                                  ip4_address_t * addr,
2991                                  u16 * port,
2992                                  u16 port_per_thread, u32 snat_thread_index)
2993 {
2994   int i;
2995   snat_address_t *a, *ga = 0;
2996   u32 portnum;
2997
2998   for (i = 0; i < vec_len (addresses); i++)
2999     {
3000       a = addresses + i;
3001       switch (proto)
3002         {
3003 #define _(N, j, n, s) \
3004         case NAT_PROTOCOL_##N: \
3005           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
3006             { \
3007               if (a->fib_index == fib_index) \
3008                 { \
3009                   while (1) \
3010                     { \
3011                       portnum = (port_per_thread * \
3012                         snat_thread_index) + \
3013                         snat_random_port(0, port_per_thread - 1) + 1024; \
3014                       if (a->busy_##n##_port_refcounts[portnum]) \
3015                         continue; \
3016                       --a->busy_##n##_port_refcounts[portnum]; \
3017                       a->busy_##n##_ports_per_thread[thread_index]++; \
3018                       a->busy_##n##_ports++; \
3019                       *addr = a->addr; \
3020                       *port = clib_host_to_net_u16(portnum); \
3021                       return 0; \
3022                     } \
3023                 } \
3024               else if (a->fib_index == ~0) \
3025                 { \
3026                   ga = a; \
3027                 } \
3028             } \
3029           break;
3030           foreach_nat_protocol
3031 #undef _
3032         default:
3033           nat_elog_info ("unknown protocol");
3034           return 1;
3035         }
3036
3037     }
3038
3039   if (ga)
3040     {
3041       a = ga;
3042       switch (proto)
3043         {
3044 #define _(N, j, n, s) \
3045         case NAT_PROTOCOL_##N: \
3046           while (1) \
3047             { \
3048               portnum = (port_per_thread * \
3049                 snat_thread_index) + \
3050                 snat_random_port(0, port_per_thread - 1) + 1024; \
3051               if (a->busy_##n##_port_refcounts[portnum]) \
3052                 continue; \
3053               ++a->busy_##n##_port_refcounts[portnum]; \
3054               a->busy_##n##_ports_per_thread[thread_index]++; \
3055               a->busy_##n##_ports++; \
3056               *addr = a->addr; \
3057               *port = clib_host_to_net_u16(portnum); \
3058               return 0; \
3059             }
3060           break;
3061           foreach_nat_protocol
3062 #undef _
3063         default:
3064           nat_elog_info ("unknown protocol");
3065           return 1;
3066         }
3067     }
3068
3069   /* Totally out of translations to use... */
3070   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
3071   return 1;
3072 }
3073
3074 static int
3075 nat_alloc_addr_and_port_mape (snat_address_t * addresses, u32 fib_index,
3076                               u32 thread_index, nat_protocol_t proto,
3077                               ip4_address_t * addr, u16 * port,
3078                               u16 port_per_thread, u32 snat_thread_index)
3079 {
3080   snat_main_t *sm = &snat_main;
3081   snat_address_t *a = addresses;
3082   u16 m, ports, portnum, A, j;
3083   m = 16 - (sm->psid_offset + sm->psid_length);
3084   ports = (1 << (16 - sm->psid_length)) - (1 << m);
3085
3086   if (!vec_len (addresses))
3087     goto exhausted;
3088
3089   switch (proto)
3090     {
3091 #define _(N, i, n, s) \
3092     case NAT_PROTOCOL_##N: \
3093       if (a->busy_##n##_ports < ports) \
3094         { \
3095           while (1) \
3096             { \
3097               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
3098               j = snat_random_port(0, pow2_mask(m)); \
3099               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
3100               if (a->busy_##n##_port_refcounts[portnum]) \
3101                 continue; \
3102               ++a->busy_##n##_port_refcounts[portnum]; \
3103               a->busy_##n##_ports++; \
3104               *addr = a->addr; \
3105               *port = clib_host_to_net_u16 (portnum); \
3106               return 0; \
3107             } \
3108         } \
3109       break;
3110       foreach_nat_protocol
3111 #undef _
3112     default:
3113       nat_elog_info ("unknown protocol");
3114       return 1;
3115     }
3116
3117 exhausted:
3118   /* Totally out of translations to use... */
3119   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
3120   return 1;
3121 }
3122
3123 static int
3124 nat_alloc_addr_and_port_range (snat_address_t * addresses, u32 fib_index,
3125                                u32 thread_index, nat_protocol_t proto,
3126                                ip4_address_t * addr, u16 * port,
3127                                u16 port_per_thread, u32 snat_thread_index)
3128 {
3129   snat_main_t *sm = &snat_main;
3130   snat_address_t *a = addresses;
3131   u16 portnum, ports;
3132
3133   ports = sm->end_port - sm->start_port + 1;
3134
3135   if (!vec_len (addresses))
3136     goto exhausted;
3137
3138   switch (proto)
3139     {
3140 #define _(N, i, n, s) \
3141     case NAT_PROTOCOL_##N: \
3142       if (a->busy_##n##_ports < ports) \
3143         { \
3144           while (1) \
3145             { \
3146               portnum = snat_random_port(sm->start_port, sm->end_port); \
3147               if (a->busy_##n##_port_refcounts[portnum]) \
3148                 continue; \
3149               ++a->busy_##n##_port_refcounts[portnum]; \
3150               a->busy_##n##_ports++; \
3151               *addr = a->addr; \
3152               *port = clib_host_to_net_u16 (portnum); \
3153               return 0; \
3154             } \
3155         } \
3156       break;
3157       foreach_nat_protocol
3158 #undef _
3159     default:
3160       nat_elog_info ("unknown protocol");
3161       return 1;
3162     }
3163
3164 exhausted:
3165   /* Totally out of translations to use... */
3166   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
3167   return 1;
3168 }
3169
3170 void
3171 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
3172 {
3173   dpo_id_t dpo_v4 = DPO_INVALID;
3174   fib_prefix_t pfx = {
3175     .fp_proto = FIB_PROTOCOL_IP4,
3176     .fp_len = 32,
3177     .fp_addr.ip4.as_u32 = addr.as_u32,
3178   };
3179
3180   if (is_add)
3181     {
3182       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
3183       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
3184                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
3185       dpo_reset (&dpo_v4);
3186     }
3187   else
3188     {
3189       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
3190     }
3191 }
3192
3193 u8 *
3194 format_session_kvp (u8 * s, va_list * args)
3195 {
3196   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3197
3198   s = format (s, "%U session-index %llu", format_snat_key, v->key, v->value);
3199
3200   return s;
3201 }
3202
3203 u8 *
3204 format_static_mapping_kvp (u8 * s, va_list * args)
3205 {
3206   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3207
3208   s = format (s, "%U static-mapping-index %llu",
3209               format_snat_key, v->key, v->value);
3210
3211   return s;
3212 }
3213
3214 u8 *
3215 format_user_kvp (u8 * s, va_list * args)
3216 {
3217   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3218   snat_user_key_t k;
3219
3220   k.as_u64 = v->key;
3221
3222   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3223               k.fib_index, v->value);
3224
3225   return s;
3226 }
3227
3228 u8 *
3229 format_ed_session_kvp (u8 * s, va_list * args)
3230 {
3231   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3232
3233   u8 proto;
3234   u16 r_port, l_port;
3235   ip4_address_t l_addr, r_addr;
3236   u32 fib_index;
3237
3238   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
3239   s =
3240     format (s,
3241             "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
3242             format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
3243             format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
3244             format_ip_protocol, proto, fib_index,
3245             ed_value_get_session_index (v), ed_value_get_thread_index (v));
3246
3247   return s;
3248 }
3249
3250 static u32
3251 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3252                            u8 is_output)
3253 {
3254   snat_main_t *sm = &snat_main;
3255   u32 next_worker_index = 0;
3256   u32 hash;
3257
3258   next_worker_index = sm->first_worker_index;
3259   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3260     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3261
3262   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3263     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3264   else
3265     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3266
3267   return next_worker_index;
3268 }
3269
3270 static u32
3271 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3272                            u32 rx_fib_index0, u8 is_output)
3273 {
3274   snat_main_t *sm = &snat_main;
3275   udp_header_t *udp;
3276   u16 port;
3277   clib_bihash_kv_8_8_t kv, value;
3278   snat_static_mapping_t *m;
3279   u32 proto;
3280   u32 next_worker_index = 0;
3281
3282   /* first try static mappings without port */
3283   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3284     {
3285       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
3286       if (!clib_bihash_search_8_8
3287           (&sm->static_mapping_by_external, &kv, &value))
3288         {
3289           m = pool_elt_at_index (sm->static_mappings, value.value);
3290           return m->workers[0];
3291         }
3292     }
3293
3294   proto = ip_proto_to_nat_proto (ip0->protocol);
3295   udp = ip4_next_header (ip0);
3296   port = udp->dst_port;
3297
3298   /* unknown protocol */
3299   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3300     {
3301       /* use current thread */
3302       return vlib_get_thread_index ();
3303     }
3304
3305   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3306     {
3307       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3308       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3309       if (!icmp_type_is_error_message
3310           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3311         port = vnet_buffer (b)->ip.reass.l4_src_port;
3312       else
3313         {
3314           /* if error message, then it's not fragmented and we can access it */
3315           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3316           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3317           void *l4_header = ip4_next_header (inner_ip);
3318           switch (proto)
3319             {
3320             case NAT_PROTOCOL_ICMP:
3321               icmp = (icmp46_header_t *) l4_header;
3322               echo = (icmp_echo_header_t *) (icmp + 1);
3323               port = echo->identifier;
3324               break;
3325             case NAT_PROTOCOL_UDP:
3326             case NAT_PROTOCOL_TCP:
3327               port = ((tcp_udp_header_t *) l4_header)->src_port;
3328               break;
3329             default:
3330               return vlib_get_thread_index ();
3331             }
3332         }
3333     }
3334
3335   /* try static mappings with port */
3336   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3337     {
3338       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
3339       if (!clib_bihash_search_8_8
3340           (&sm->static_mapping_by_external, &kv, &value))
3341         {
3342           m = pool_elt_at_index (sm->static_mappings, value.value);
3343           return m->workers[0];
3344         }
3345     }
3346
3347   /* worker by outside port */
3348   next_worker_index = sm->first_worker_index;
3349   next_worker_index +=
3350     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3351   return next_worker_index;
3352 }
3353
3354 static u32
3355 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3356                                u8 is_output)
3357 {
3358   snat_main_t *sm = &snat_main;
3359   u32 next_worker_index = sm->first_worker_index;
3360   u32 hash;
3361
3362   clib_bihash_kv_16_8_t kv16, value16;
3363   snat_main_per_thread_data_t *tsm;
3364   udp_header_t *udp;
3365
3366   if (PREDICT_FALSE (is_output))
3367     {
3368       u32 fib_index = sm->outside_fib_index;
3369       nat_outside_fib_t *outside_fib;
3370       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3371       fib_prefix_t pfx = {
3372         .fp_proto = FIB_PROTOCOL_IP4,
3373         .fp_len = 32,
3374         .fp_addr = {
3375                     .ip4.as_u32 = ip->dst_address.as_u32,
3376                     }
3377         ,
3378       };
3379
3380       udp = ip4_next_header (ip);
3381
3382       switch (vec_len (sm->outside_fibs))
3383         {
3384         case 0:
3385           fib_index = sm->outside_fib_index;
3386           break;
3387         case 1:
3388           fib_index = sm->outside_fibs[0].fib_index;
3389           break;
3390         default:
3391             /* *INDENT-OFF* */
3392             vec_foreach (outside_fib, sm->outside_fibs)
3393               {
3394                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3395                 if (FIB_NODE_INDEX_INVALID != fei)
3396                   {
3397                     if (fib_entry_get_resolving_interface (fei) != ~0)
3398                       {
3399                         fib_index = outside_fib->fib_index;
3400                         break;
3401                       }
3402                   }
3403               }
3404             /* *INDENT-ON* */
3405           break;
3406         }
3407
3408       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
3409                  udp->dst_port, fib_index, ip->protocol);
3410
3411       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3412                                                   &kv16, &value16)))
3413         {
3414           tsm =
3415             vec_elt_at_index (sm->per_thread_data,
3416                               ed_value_get_thread_index (&value16));
3417           next_worker_index += tsm->thread_index;
3418
3419           nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3420                                   next_worker_index, fib_index,
3421                                   clib_net_to_host_u32 (ip->
3422                                                         src_address.as_u32),
3423                                   clib_net_to_host_u32 (ip->
3424                                                         dst_address.as_u32));
3425
3426           return next_worker_index;
3427         }
3428     }
3429
3430   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3431     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3432
3433   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3434     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3435   else
3436     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3437
3438   if (PREDICT_TRUE (!is_output))
3439     {
3440       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3441                               next_worker_index, rx_fib_index,
3442                               clib_net_to_host_u32 (ip->src_address.as_u32),
3443                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3444     }
3445   else
3446     {
3447       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3448                               next_worker_index, rx_fib_index,
3449                               clib_net_to_host_u32 (ip->src_address.as_u32),
3450                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3451     }
3452
3453   return next_worker_index;
3454 }
3455
3456 static u32
3457 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3458                                u32 rx_fib_index, u8 is_output)
3459 {
3460   snat_main_t *sm = &snat_main;
3461   clib_bihash_kv_8_8_t kv, value;
3462   clib_bihash_kv_16_8_t kv16, value16;
3463   snat_main_per_thread_data_t *tsm;
3464
3465   u32 proto, next_worker_index = 0;
3466   udp_header_t *udp;
3467   u16 port;
3468   snat_static_mapping_t *m;
3469   u32 hash;
3470
3471   proto = ip_proto_to_nat_proto (ip->protocol);
3472
3473   if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
3474     {
3475       udp = ip4_next_header (ip);
3476
3477       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
3478                  udp->src_port, rx_fib_index, ip->protocol);
3479
3480       if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3481                                                   &kv16, &value16)))
3482         {
3483           tsm =
3484             vec_elt_at_index (sm->per_thread_data,
3485                               ed_value_get_thread_index (&value16));
3486           vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
3487             ed_value_get_session_index (&value16);
3488           next_worker_index = sm->first_worker_index + tsm->thread_index;
3489           nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3490                                   next_worker_index, rx_fib_index,
3491                                   clib_net_to_host_u32 (ip->
3492                                                         src_address.as_u32),
3493                                   clib_net_to_host_u32 (ip->
3494                                                         dst_address.as_u32));
3495           return next_worker_index;
3496         }
3497     }
3498   else if (proto == NAT_PROTOCOL_ICMP)
3499     {
3500       if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
3501         {
3502           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
3503                                                       &kv16, &value16)))
3504             {
3505               tsm =
3506                 vec_elt_at_index (sm->per_thread_data,
3507                                   ed_value_get_thread_index (&value16));
3508               next_worker_index = sm->first_worker_index + tsm->thread_index;
3509               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3510                                       next_worker_index, rx_fib_index,
3511                                       clib_net_to_host_u32 (ip->
3512                                                             src_address.as_u32),
3513                                       clib_net_to_host_u32 (ip->
3514                                                             dst_address.as_u32));
3515               return next_worker_index;
3516             }
3517         }
3518     }
3519
3520   /* first try static mappings without port */
3521   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3522     {
3523       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
3524       if (!clib_bihash_search_8_8
3525           (&sm->static_mapping_by_external, &kv, &value))
3526         {
3527           m = pool_elt_at_index (sm->static_mappings, value.value);
3528           next_worker_index = m->workers[0];
3529           goto done;
3530         }
3531     }
3532
3533   /* unknown protocol */
3534   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
3535     {
3536       /* use current thread */
3537       next_worker_index = vlib_get_thread_index ();
3538       goto done;
3539     }
3540
3541   udp = ip4_next_header (ip);
3542   port = udp->dst_port;
3543
3544   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3545     {
3546       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3547       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3548       if (!icmp_type_is_error_message
3549           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3550         port = vnet_buffer (b)->ip.reass.l4_src_port;
3551       else
3552         {
3553           /* if error message, then it's not fragmented and we can access it */
3554           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3555           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3556           void *l4_header = ip4_next_header (inner_ip);
3557           switch (proto)
3558             {
3559             case NAT_PROTOCOL_ICMP:
3560               icmp = (icmp46_header_t *) l4_header;
3561               echo = (icmp_echo_header_t *) (icmp + 1);
3562               port = echo->identifier;
3563               break;
3564             case NAT_PROTOCOL_UDP:
3565             case NAT_PROTOCOL_TCP:
3566               port = ((tcp_udp_header_t *) l4_header)->src_port;
3567               break;
3568             default:
3569               next_worker_index = vlib_get_thread_index ();
3570               goto done;
3571             }
3572         }
3573     }
3574
3575   /* try static mappings with port */
3576   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3577     {
3578       init_nat_k (&kv, ip->dst_address, proto, 0, port);
3579       if (!clib_bihash_search_8_8
3580           (&sm->static_mapping_by_external, &kv, &value))
3581         {
3582           m = pool_elt_at_index (sm->static_mappings, value.value);
3583           if (!is_lb_static_mapping (m))
3584             {
3585               next_worker_index = m->workers[0];
3586               goto done;
3587             }
3588
3589           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3590             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3591
3592           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3593             next_worker_index =
3594               m->workers[hash & (_vec_len (m->workers) - 1)];
3595           else
3596             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3597           goto done;
3598         }
3599     }
3600
3601   /* worker by outside port */
3602   next_worker_index = sm->first_worker_index;
3603   next_worker_index +=
3604     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3605
3606 done:
3607   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3608                           clib_net_to_host_u32 (ip->src_address.as_u32),
3609                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3610   return next_worker_index;
3611 }
3612
3613 void
3614 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3615                 ip4_address_t * out_addr, u16 out_port,
3616                 ip4_address_t * eh_addr, u16 eh_port,
3617                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3618                 u32 fib_index, u16 flags, u32 thread_index)
3619 {
3620   snat_main_t *sm = &snat_main;
3621   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3622   snat_user_t *u;
3623   snat_session_t *s;
3624   clib_bihash_kv_8_8_t kv;
3625   vlib_main_t *vm = vlib_get_main ();
3626   f64 now = vlib_time_now (vm);
3627   nat_outside_fib_t *outside_fib;
3628   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3629   fib_prefix_t pfx = {
3630     .fp_proto = FIB_PROTOCOL_IP4,
3631     .fp_len = 32,
3632     .fp_addr = {
3633                 .ip4.as_u32 = eh_addr->as_u32,
3634                 },
3635   };
3636
3637   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3638     {
3639       if (nat_set_outside_address_and_port
3640           (sm->addresses, thread_index, *out_addr, out_port, proto))
3641         return;
3642     }
3643
3644   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3645   if (!u)
3646     return;
3647
3648   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3649   if (!s)
3650     return;
3651
3652   if (sm->endpoint_dependent)
3653     {
3654       nat_ed_lru_insert (tsm, s, now, nat_proto_to_ip_proto (proto));
3655     }
3656
3657   s->out2in.addr.as_u32 = out_addr->as_u32;
3658   s->out2in.port = out_port;
3659   s->nat_proto = proto;
3660   s->last_heard = now;
3661   s->flags = flags;
3662   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3663   s->ext_host_port = eh_port;
3664   user_session_increment (sm, u, snat_is_session_static (s));
3665   switch (vec_len (sm->outside_fibs))
3666     {
3667     case 0:
3668       s->out2in.fib_index = sm->outside_fib_index;
3669       break;
3670     case 1:
3671       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3672       break;
3673     default:
3674       /* *INDENT-OFF* */
3675       vec_foreach (outside_fib, sm->outside_fibs)
3676         {
3677           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3678           if (FIB_NODE_INDEX_INVALID != fei)
3679             {
3680               if (fib_entry_get_resolving_interface (fei) != ~0)
3681                 {
3682                   s->out2in.fib_index = outside_fib->fib_index;
3683                   break;
3684                 }
3685             }
3686         }
3687       /* *INDENT-ON* */
3688       break;
3689     }
3690   init_nat_o2i_kv (&kv, s, s - tsm->sessions);
3691   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3692     nat_elog_warn ("out2in key add failed");
3693
3694   s->in2out.addr.as_u32 = in_addr->as_u32;
3695   s->in2out.port = in_port;
3696   s->in2out.fib_index = fib_index;
3697   init_nat_i2o_kv (&kv, s, s - tsm->sessions);
3698   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3699     nat_elog_warn ("in2out key add failed");
3700 }
3701
3702 void
3703 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3704                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3705                 u32 ti)
3706 {
3707   snat_main_t *sm = &snat_main;
3708   clib_bihash_kv_8_8_t kv, value;
3709   u32 thread_index;
3710   snat_session_t *s;
3711   snat_main_per_thread_data_t *tsm;
3712
3713   if (sm->num_workers > 1)
3714     thread_index =
3715       sm->first_worker_index +
3716       (sm->workers[(clib_net_to_host_u16 (out_port) -
3717                     1024) / sm->port_per_thread]);
3718   else
3719     thread_index = sm->num_workers;
3720   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3721
3722   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3723   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3724     return;
3725
3726   s = pool_elt_at_index (tsm->sessions, value.value);
3727   nat_free_session_data (sm, s, thread_index, 1);
3728   nat44_delete_session (sm, s, thread_index);
3729 }
3730
3731 void
3732 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3733                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3734                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3735 {
3736   snat_main_t *sm = &snat_main;
3737   clib_bihash_kv_8_8_t kv, value;
3738   snat_session_t *s;
3739   snat_main_per_thread_data_t *tsm;
3740
3741   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3742
3743   init_nat_k (&kv, *out_addr, out_port, fib_index, proto);
3744   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3745     return;
3746
3747   s = pool_elt_at_index (tsm->sessions, value.value);
3748   s->total_pkts = total_pkts;
3749   s->total_bytes = total_bytes;
3750 }
3751
3752 void
3753 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3754                    ip4_address_t * out_addr, u16 out_port,
3755                    ip4_address_t * eh_addr, u16 eh_port,
3756                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3757                    u32 fib_index, u16 flags, u32 thread_index)
3758 {
3759   snat_main_t *sm = &snat_main;
3760   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3761   snat_session_t *s;
3762   clib_bihash_kv_16_8_t kv;
3763   vlib_main_t *vm = vlib_get_main ();
3764   f64 now = vlib_time_now (vm);
3765   nat_outside_fib_t *outside_fib;
3766   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3767   fib_prefix_t pfx = {
3768     .fp_proto = FIB_PROTOCOL_IP4,
3769     .fp_len = 32,
3770     .fp_addr = {
3771                 .ip4.as_u32 = eh_addr->as_u32,
3772                 },
3773   };
3774
3775
3776   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3777     {
3778       if (nat_set_outside_address_and_port
3779           (sm->addresses, thread_index, *out_addr, out_port, proto))
3780         return;
3781     }
3782
3783   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3784     {
3785       if (nat_set_outside_address_and_port
3786           (sm->addresses, thread_index, *ehn_addr, ehn_port, proto))
3787         return;
3788     }
3789
3790   s = nat_ed_session_alloc (sm, thread_index, now, proto);
3791   if (!s)
3792     return;
3793
3794   s->last_heard = now;
3795   s->flags = flags;
3796   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3797   s->ext_host_nat_port = s->ext_host_port = eh_port;
3798   if (is_twice_nat_session (s))
3799     {
3800       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3801       s->ext_host_nat_port = ehn_port;
3802     }
3803   switch (vec_len (sm->outside_fibs))
3804     {
3805     case 0:
3806       s->out2in.fib_index = sm->outside_fib_index;
3807       break;
3808     case 1:
3809       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
3810       break;
3811     default:
3812       /* *INDENT-OFF* */
3813       vec_foreach (outside_fib, sm->outside_fibs)
3814         {
3815           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3816           if (FIB_NODE_INDEX_INVALID != fei)
3817             {
3818               if (fib_entry_get_resolving_interface (fei) != ~0)
3819                 {
3820                   s->out2in.fib_index = outside_fib->fib_index;
3821                   break;
3822                 }
3823             }
3824         }
3825       /* *INDENT-ON* */
3826       break;
3827     }
3828   s->nat_proto = proto;
3829   s->out2in.addr.as_u32 = out_addr->as_u32;
3830   s->out2in.port = out_port;
3831
3832   s->in2out.addr.as_u32 = in_addr->as_u32;
3833   s->in2out.port = in_port;
3834   s->in2out.fib_index = fib_index;
3835
3836   init_ed_kv (&kv, *in_addr, in_port, s->ext_host_nat_addr,
3837               s->ext_host_nat_port, fib_index, nat_proto_to_ip_proto (proto),
3838               thread_index, s - tsm->sessions);
3839   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3840     nat_elog_warn ("in2out key add failed");
3841
3842   init_ed_kv (&kv, *out_addr, out_port, *eh_addr, eh_port,
3843               s->out2in.fib_index, nat_proto_to_ip_proto (proto),
3844               thread_index, s - tsm->sessions);
3845   if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 1))
3846     nat_elog_warn ("out2in key add failed");
3847 }
3848
3849 void
3850 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3851                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3852                    u32 fib_index, u32 ti)
3853 {
3854   snat_main_t *sm = &snat_main;
3855   clib_bihash_kv_16_8_t kv, value;
3856   u32 thread_index;
3857   snat_session_t *s;
3858   snat_main_per_thread_data_t *tsm;
3859
3860   if (sm->num_workers > 1)
3861     thread_index =
3862       sm->first_worker_index +
3863       (sm->workers[(clib_net_to_host_u16 (out_port) -
3864                     1024) / sm->port_per_thread]);
3865   else
3866     thread_index = sm->num_workers;
3867   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3868
3869   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3870   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3871     return;
3872
3873   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3874   nat_free_session_data (sm, s, thread_index, 1);
3875   nat44_delete_session (sm, s, thread_index);
3876 }
3877
3878 void
3879 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3880                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3881                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3882                    u32 thread_index)
3883 {
3884   snat_main_t *sm = &snat_main;
3885   clib_bihash_kv_16_8_t kv, value;
3886   snat_session_t *s;
3887   snat_main_per_thread_data_t *tsm;
3888
3889   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3890
3891   init_ed_k (&kv, *out_addr, out_port, *eh_addr, eh_port, fib_index, proto);
3892   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
3893     return;
3894
3895   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3896   s->total_pkts = total_pkts;
3897   s->total_bytes = total_bytes;
3898 }
3899
3900 static u32
3901 nat_calc_bihash_buckets (u32 n_elts)
3902 {
3903   return 1 << (max_log2 (n_elts >> 1) + 1);
3904 }
3905
3906 static u32
3907 nat_calc_bihash_memory (u32 n_buckets, uword kv_size)
3908 {
3909   return n_buckets * (8 + kv_size * 4);
3910 }
3911
3912 u32
3913 nat44_get_max_session_limit ()
3914 {
3915   snat_main_t *sm = &snat_main;
3916   u32 max_limit = 0, len = 0;
3917
3918   for (; len < vec_len (sm->max_translations_per_fib); len++)
3919     {
3920       if (max_limit < sm->max_translations_per_fib[len])
3921         max_limit = sm->max_translations_per_fib[len];
3922     }
3923   return max_limit;
3924 }
3925
3926 int
3927 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3928 {
3929   snat_main_t *sm = &snat_main;
3930   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3931   u32 len = vec_len (sm->max_translations_per_fib);
3932
3933   if (len <= fib_index)
3934     {
3935       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3936
3937       for (; len < vec_len (sm->max_translations_per_fib); len++)
3938         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3939     }
3940
3941   sm->max_translations_per_fib[fib_index] = session_limit;
3942   return 0;
3943 }
3944
3945 int
3946 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3947 {
3948   snat_main_t *sm = &snat_main;
3949
3950   if (nat44_set_session_limit (session_limit, vrf_id))
3951     return 1;
3952   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3953
3954   sm->translation_buckets =
3955     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3956
3957   if (!sm->translation_memory_size_set)
3958     {
3959       sm->translation_memory_size =
3960         nat_calc_bihash_memory (sm->translation_buckets,
3961                                 sizeof (clib_bihash_16_8_t));
3962     }
3963
3964   nat44_sessions_clear ();
3965   return 0;
3966 }
3967
3968 void
3969 nat44_db_init (snat_main_per_thread_data_t * tsm)
3970 {
3971   snat_main_t *sm = &snat_main;
3972
3973   pool_alloc (tsm->sessions, sm->max_translations_per_thread);
3974   pool_alloc (tsm->lru_pool, sm->max_translations_per_thread);
3975
3976   dlist_elt_t *head;
3977
3978   pool_get (tsm->lru_pool, head);
3979   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3980   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3981
3982   pool_get (tsm->lru_pool, head);
3983   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3984   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3985
3986   pool_get (tsm->lru_pool, head);
3987   tsm->udp_lru_head_index = head - tsm->lru_pool;
3988   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3989
3990   pool_get (tsm->lru_pool, head);
3991   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3992   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3993
3994   pool_get (tsm->lru_pool, head);
3995   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3996   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3997
3998   if (sm->endpoint_dependent)
3999     {
4000       clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
4001                              sm->translation_buckets,
4002                              sm->translation_memory_size);
4003       clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
4004                                           format_ed_session_kvp);
4005
4006     }
4007   else
4008     {
4009       clib_bihash_init_8_8 (&tsm->in2out, "in2out",
4010                             sm->translation_buckets,
4011                             sm->translation_memory_size);
4012       clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out, format_session_kvp);
4013       clib_bihash_init_8_8 (&tsm->out2in, "out2in",
4014                             sm->translation_buckets,
4015                             sm->translation_memory_size);
4016       clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in, format_session_kvp);
4017     }
4018
4019   // TODO: resolve static mappings (put only to !ED)
4020   pool_alloc (tsm->list_pool, sm->max_translations_per_thread);
4021   clib_bihash_init_8_8 (&tsm->user_hash, "users", sm->user_buckets,
4022                         sm->user_memory_size);
4023   clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash, format_user_kvp);
4024 }
4025
4026 void
4027 nat44_db_free (snat_main_per_thread_data_t * tsm)
4028 {
4029   snat_main_t *sm = &snat_main;
4030
4031   pool_free (tsm->sessions);
4032   pool_free (tsm->lru_pool);
4033
4034   if (sm->endpoint_dependent)
4035     {
4036       clib_bihash_free_16_8 (&tsm->in2out_ed);
4037       vec_free (tsm->per_vrf_sessions_vec);
4038     }
4039   else
4040     {
4041       clib_bihash_free_8_8 (&tsm->in2out);
4042       clib_bihash_free_8_8 (&tsm->out2in);
4043     }
4044
4045   // TODO: resolve static mappings (put only to !ED)
4046   pool_free (tsm->users);
4047   pool_free (tsm->list_pool);
4048   clib_bihash_free_8_8 (&tsm->user_hash);
4049 }
4050
4051 void
4052 nat44_sessions_clear ()
4053 {
4054   snat_main_t *sm = &snat_main;
4055   snat_main_per_thread_data_t *tsm;
4056
4057   if (sm->endpoint_dependent)
4058     {
4059       clib_bihash_free_16_8 (&sm->out2in_ed);
4060       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4061                              clib_max (1, sm->num_workers) *
4062                              sm->translation_buckets,
4063                              clib_max (1, sm->num_workers) *
4064                              sm->translation_memory_size);
4065       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4066                                           format_ed_session_kvp);
4067     }
4068
4069   /* *INDENT-OFF* */
4070   vec_foreach (tsm, sm->per_thread_data)
4071     {
4072       u32 ti;
4073
4074       nat44_db_free (tsm);
4075       nat44_db_init (tsm);
4076
4077       ti = tsm->snat_thread_index;
4078       vlib_set_simple_counter (&sm->total_users, ti, 0, 0);
4079       vlib_set_simple_counter (&sm->total_sessions, ti, 0, 0);
4080     }
4081   /* *INDENT-ON* */
4082 }
4083
4084 static clib_error_t *
4085 snat_config (vlib_main_t * vm, unformat_input_t * input)
4086 {
4087   snat_main_t *sm = &snat_main;
4088   snat_main_per_thread_data_t *tsm;
4089
4090   u32 static_mapping_buckets = 1024;
4091   uword static_mapping_memory_size = 64 << 20;
4092
4093   u32 max_users_per_thread = 0;
4094   u32 user_memory_size = 0;
4095   u32 max_translations_per_thread = 0;
4096   u32 translation_memory_size = 0;
4097
4098   u32 max_translations_per_user = ~0;
4099
4100   u32 outside_vrf_id = 0;
4101   u32 outside_ip6_vrf_id = 0;
4102   u32 inside_vrf_id = 0;
4103   u8 static_mapping_only = 0;
4104   u8 static_mapping_connection_tracking = 0;
4105
4106   u32 udp_timeout = SNAT_UDP_TIMEOUT;
4107   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
4108   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
4109   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
4110
4111   sm->out2in_dpo = 0;
4112   sm->endpoint_dependent = 0;
4113
4114   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
4115     {
4116       if (unformat
4117           (input, "max translations per thread %d",
4118            &max_translations_per_thread))
4119         ;
4120       else if (unformat (input, "udp timeout %d", &udp_timeout))
4121         ;
4122       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
4123         ;
4124       else if (unformat (input, "tcp transitory timeout %d",
4125                          &tcp_transitory_timeout));
4126       else if (unformat (input, "tcp established timeout %d",
4127                          &tcp_established_timeout));
4128       else if (unformat (input, "translation hash memory %d",
4129                          &translation_memory_size));
4130       else
4131         if (unformat
4132             (input, "max users per thread %d", &max_users_per_thread))
4133         ;
4134       else if (unformat (input, "user hash memory %d", &user_memory_size))
4135         ;
4136       else if (unformat (input, "max translations per user %d",
4137                          &max_translations_per_user))
4138         ;
4139       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
4140         ;
4141       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
4142         ;
4143       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
4144         ;
4145       else if (unformat (input, "static mapping only"))
4146         {
4147           static_mapping_only = 1;
4148           if (unformat (input, "connection tracking"))
4149             static_mapping_connection_tracking = 1;
4150         }
4151       else if (unformat (input, "out2in dpo"))
4152         sm->out2in_dpo = 1;
4153       else if (unformat (input, "endpoint-dependent"))
4154         sm->endpoint_dependent = 1;
4155       else
4156         return clib_error_return (0, "unknown input '%U'",
4157                                   format_unformat_error, input);
4158     }
4159
4160   if (static_mapping_only && (sm->endpoint_dependent))
4161     return clib_error_return (0,
4162                               "static mapping only mode available only for simple nat");
4163
4164   if (sm->out2in_dpo && (sm->endpoint_dependent))
4165     return clib_error_return (0,
4166                               "out2in dpo mode available only for simple nat");
4167   if (sm->endpoint_dependent && max_users_per_thread > 0)
4168     {
4169       return clib_error_return (0,
4170                                 "setting 'max users' in endpoint-dependent mode is not supported");
4171     }
4172
4173   if (sm->endpoint_dependent && max_translations_per_user != ~0)
4174     {
4175       return clib_error_return (0,
4176                                 "setting 'max translations per user' in endpoint-dependent mode is not supported");
4177     }
4178
4179   /* optionally configurable timeouts for testing purposes */
4180   sm->udp_timeout = udp_timeout;
4181   sm->tcp_transitory_timeout = tcp_transitory_timeout;
4182   sm->tcp_established_timeout = tcp_established_timeout;
4183   sm->icmp_timeout = icmp_timeout;
4184
4185   if (0 == max_users_per_thread)
4186     {
4187       max_users_per_thread = 1024;
4188     }
4189   sm->max_users_per_thread = max_users_per_thread;
4190   sm->user_buckets = nat_calc_bihash_buckets (sm->max_users_per_thread);
4191
4192   if (0 == max_translations_per_thread)
4193     {
4194       // default value based on legacy setting of load factor 10 * default
4195       // translation buckets 1024
4196       max_translations_per_thread = 10 * 1024;
4197     }
4198   sm->translation_memory_size_set = translation_memory_size != 0;
4199
4200   sm->max_translations_per_thread = max_translations_per_thread;
4201   sm->translation_buckets =
4202     nat_calc_bihash_buckets (sm->max_translations_per_thread);
4203   if (0 == translation_memory_size)
4204     {
4205       translation_memory_size =
4206         nat_calc_bihash_memory (sm->translation_buckets,
4207                                 sizeof (clib_bihash_16_8_t));
4208     }
4209   sm->translation_memory_size = translation_memory_size;
4210   if (0 == user_memory_size)
4211     {
4212       user_memory_size =
4213         nat_calc_bihash_memory (sm->max_users_per_thread,
4214                                 sizeof (clib_bihash_8_8_t));
4215     }
4216   sm->user_memory_size = user_memory_size;
4217   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
4218
4219   sm->max_translations_per_user = max_translations_per_user == ~0 ?
4220     sm->max_translations_per_thread : max_translations_per_user;
4221
4222   sm->outside_vrf_id = outside_vrf_id;
4223   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4224                                                              outside_vrf_id,
4225                                                              nat_fib_src_hi);
4226   sm->inside_vrf_id = inside_vrf_id;
4227   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
4228                                                             inside_vrf_id,
4229                                                             nat_fib_src_hi);
4230   sm->static_mapping_only = static_mapping_only;
4231   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
4232
4233   if (sm->endpoint_dependent)
4234     {
4235       sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
4236       sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
4237
4238       sm->in2out_node_index = nat44_ed_in2out_node.index;
4239       sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
4240       sm->out2in_node_index = nat44_ed_out2in_node.index;
4241
4242       sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
4243       sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
4244       nat_affinity_init (vm);
4245       nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
4246                    nat_ha_sref_ed_cb);
4247       clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
4248                              clib_max (1, sm->num_workers) *
4249                              sm->translation_buckets,
4250                              clib_max (1, sm->num_workers) *
4251                              sm->translation_memory_size);
4252       clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed,
4253                                           format_ed_session_kvp);
4254     }
4255   else
4256     {
4257       sm->worker_in2out_cb = snat_get_worker_in2out_cb;
4258       sm->worker_out2in_cb = snat_get_worker_out2in_cb;
4259
4260       sm->in2out_node_index = snat_in2out_node.index;
4261       sm->in2out_output_node_index = snat_in2out_output_node.index;
4262       sm->out2in_node_index = snat_out2in_node.index;
4263
4264       sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
4265       sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
4266       nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
4267     }
4268   if (!static_mapping_only ||
4269       (static_mapping_only && static_mapping_connection_tracking))
4270     {
4271           /* *INDENT-OFF* */
4272           vec_foreach (tsm, sm->per_thread_data)
4273             {
4274               nat44_db_init (tsm);
4275             }
4276           /* *INDENT-ON* */
4277     }
4278   else
4279     {
4280       sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
4281       sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
4282     }
4283   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
4284                         "static_mapping_by_local", static_mapping_buckets,
4285                         static_mapping_memory_size);
4286   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
4287                                      format_static_mapping_kvp);
4288
4289   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
4290                         "static_mapping_by_external",
4291                         static_mapping_buckets, static_mapping_memory_size);
4292   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
4293                                      format_static_mapping_kvp);
4294
4295   return 0;
4296 }
4297
4298 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4299
4300 static void
4301 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4302                                  uword opaque,
4303                                  u32 sw_if_index,
4304                                  ip4_address_t * address,
4305                                  u32 address_length,
4306                                  u32 if_address_index, u32 is_delete)
4307 {
4308   snat_main_t *sm = &snat_main;
4309   snat_static_map_resolve_t *rp;
4310   snat_static_mapping_t *m;
4311   clib_bihash_kv_8_8_t kv, value;
4312   int i, rv;
4313   ip4_address_t l_addr;
4314
4315   for (i = 0; i < vec_len (sm->to_resolve); i++)
4316     {
4317       rp = sm->to_resolve + i;
4318       if (rp->addr_only == 0)
4319         continue;
4320       if (rp->sw_if_index == sw_if_index)
4321         goto match;
4322     }
4323
4324   return;
4325
4326 match:
4327   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
4328               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
4329   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4330     m = 0;
4331   else
4332     m = pool_elt_at_index (sm->static_mappings, value.value);
4333
4334   if (!is_delete)
4335     {
4336       /* Don't trip over lease renewal, static config */
4337       if (m)
4338         return;
4339     }
4340   else
4341     {
4342       if (!m)
4343         return;
4344     }
4345
4346   /* Indetity mapping? */
4347   if (rp->l_addr.as_u32 == 0)
4348     l_addr.as_u32 = address[0].as_u32;
4349   else
4350     l_addr.as_u32 = rp->l_addr.as_u32;
4351   /* Add the static mapping */
4352   rv = snat_add_static_mapping (l_addr,
4353                                 address[0],
4354                                 rp->l_port,
4355                                 rp->e_port,
4356                                 rp->vrf_id,
4357                                 rp->addr_only, ~0 /* sw_if_index */ ,
4358                                 rp->proto, !is_delete, rp->twice_nat,
4359                                 rp->out2in_only, rp->tag, rp->identity_nat,
4360                                 rp->pool_addr, rp->exact);
4361   if (rv)
4362     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4363 }
4364
4365 static void
4366 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4367                                        uword opaque,
4368                                        u32 sw_if_index,
4369                                        ip4_address_t * address,
4370                                        u32 address_length,
4371                                        u32 if_address_index, u32 is_delete)
4372 {
4373   snat_main_t *sm = &snat_main;
4374   snat_static_map_resolve_t *rp;
4375   ip4_address_t l_addr;
4376   int i, j;
4377   int rv;
4378   u8 twice_nat = 0;
4379   snat_address_t *addresses = sm->addresses;
4380
4381   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4382     {
4383       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4384         goto match;
4385     }
4386
4387   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4388     {
4389       twice_nat = 1;
4390       addresses = sm->twice_nat_addresses;
4391       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4392         goto match;
4393     }
4394
4395   return;
4396
4397 match:
4398   if (!is_delete)
4399     {
4400       /* Don't trip over lease renewal, static config */
4401       for (j = 0; j < vec_len (addresses); j++)
4402         if (addresses[j].addr.as_u32 == address->as_u32)
4403           return;
4404
4405       (void) snat_add_address (sm, address, ~0, twice_nat);
4406       /* Scan static map resolution vector */
4407       for (j = 0; j < vec_len (sm->to_resolve); j++)
4408         {
4409           rp = sm->to_resolve + j;
4410           if (rp->addr_only)
4411             continue;
4412           /* On this interface? */
4413           if (rp->sw_if_index == sw_if_index)
4414             {
4415               /* Indetity mapping? */
4416               if (rp->l_addr.as_u32 == 0)
4417                 l_addr.as_u32 = address[0].as_u32;
4418               else
4419                 l_addr.as_u32 = rp->l_addr.as_u32;
4420               /* Add the static mapping */
4421               rv = snat_add_static_mapping (l_addr,
4422                                             address[0],
4423                                             rp->l_port,
4424                                             rp->e_port,
4425                                             rp->vrf_id,
4426                                             rp->addr_only,
4427                                             ~0 /* sw_if_index */ ,
4428                                             rp->proto,
4429                                             rp->is_add, rp->twice_nat,
4430                                             rp->out2in_only, rp->tag,
4431                                             rp->identity_nat,
4432                                             rp->pool_addr, rp->exact);
4433               if (rv)
4434                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4435                                     "i4", rv);
4436             }
4437         }
4438       return;
4439     }
4440   else
4441     {
4442       (void) snat_del_address (sm, address[0], 1, twice_nat);
4443       return;
4444     }
4445 }
4446
4447
4448 int
4449 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4450                             u8 twice_nat)
4451 {
4452   ip4_main_t *ip4_main = sm->ip4_main;
4453   ip4_address_t *first_int_addr;
4454   snat_static_map_resolve_t *rp;
4455   u32 *indices_to_delete = 0;
4456   int i, j;
4457   u32 *auto_add_sw_if_indices =
4458     twice_nat ? sm->
4459     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4460
4461   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4462     );
4463
4464   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4465     {
4466       if (auto_add_sw_if_indices[i] == sw_if_index)
4467         {
4468           if (is_del)
4469             {
4470               /* if have address remove it */
4471               if (first_int_addr)
4472                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4473               else
4474                 {
4475                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4476                     {
4477                       rp = sm->to_resolve + j;
4478                       if (rp->sw_if_index == sw_if_index)
4479                         vec_add1 (indices_to_delete, j);
4480                     }
4481                   if (vec_len (indices_to_delete))
4482                     {
4483                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4484                         vec_del1 (sm->to_resolve, j);
4485                       vec_free (indices_to_delete);
4486                     }
4487                 }
4488               if (twice_nat)
4489                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4490               else
4491                 vec_del1 (sm->auto_add_sw_if_indices, i);
4492             }
4493           else
4494             return VNET_API_ERROR_VALUE_EXIST;
4495
4496           return 0;
4497         }
4498     }
4499
4500   if (is_del)
4501     return VNET_API_ERROR_NO_SUCH_ENTRY;
4502
4503   /* add to the auto-address list */
4504   if (twice_nat)
4505     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4506   else
4507     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4508
4509   /* If the address is already bound - or static - add it now */
4510   if (first_int_addr)
4511     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4512
4513   return 0;
4514 }
4515
4516 int
4517 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4518                    nat_protocol_t proto, u32 vrf_id, int is_in)
4519 {
4520   snat_main_per_thread_data_t *tsm;
4521   clib_bihash_kv_8_8_t kv, value;
4522   ip4_header_t ip;
4523   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4524   snat_session_t *s;
4525   clib_bihash_8_8_t *t;
4526
4527   if (sm->endpoint_dependent)
4528     return VNET_API_ERROR_UNSUPPORTED;
4529
4530   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4531   if (sm->num_workers > 1)
4532     tsm =
4533       vec_elt_at_index (sm->per_thread_data,
4534                         sm->worker_in2out_cb (&ip, fib_index, 0));
4535   else
4536     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4537
4538   init_nat_k (&kv, *addr, port, fib_index, proto);
4539   t = is_in ? &tsm->in2out : &tsm->out2in;
4540   if (!clib_bihash_search_8_8 (t, &kv, &value))
4541     {
4542       if (pool_is_free_index (tsm->sessions, value.value))
4543         return VNET_API_ERROR_UNSPECIFIED;
4544
4545       s = pool_elt_at_index (tsm->sessions, value.value);
4546       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4547       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4548       return 0;
4549     }
4550
4551   return VNET_API_ERROR_NO_SUCH_ENTRY;
4552 }
4553
4554 int
4555 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4556                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4557                       u32 vrf_id, int is_in)
4558 {
4559   ip4_header_t ip;
4560   clib_bihash_16_8_t *t;
4561   clib_bihash_kv_16_8_t kv, value;
4562   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4563   snat_session_t *s;
4564   snat_main_per_thread_data_t *tsm;
4565
4566   if (!sm->endpoint_dependent)
4567     return VNET_API_ERROR_FEATURE_DISABLED;
4568
4569   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4570   if (sm->num_workers > 1)
4571     tsm =
4572       vec_elt_at_index (sm->per_thread_data,
4573                         sm->worker_in2out_cb (&ip, fib_index, 0));
4574   else
4575     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4576
4577   t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
4578   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
4579   if (clib_bihash_search_16_8 (t, &kv, &value))
4580     {
4581       return VNET_API_ERROR_NO_SUCH_ENTRY;
4582     }
4583
4584   if (pool_is_free_index (tsm->sessions, value.value))
4585     return VNET_API_ERROR_UNSPECIFIED;
4586   s = pool_elt_at_index (tsm->sessions, value.value);
4587   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4588   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
4589   return 0;
4590 }
4591
4592 void
4593 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4594 {
4595   snat_main_t *sm = &snat_main;
4596
4597   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4598   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4599   sm->psid = psid;
4600   sm->psid_offset = psid_offset;
4601   sm->psid_length = psid_length;
4602 }
4603
4604 void
4605 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4606 {
4607   snat_main_t *sm = &snat_main;
4608
4609   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4610   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4611   sm->start_port = start_port;
4612   sm->end_port = end_port;
4613 }
4614
4615 void
4616 nat_set_alloc_addr_and_port_default (void)
4617 {
4618   snat_main_t *sm = &snat_main;
4619
4620   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4621   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4622 }
4623
4624 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4625                                  vlib_node_runtime_t * node,
4626                                  vlib_frame_t * frame)
4627 {
4628   return 0;
4629 }
4630
4631 /* *INDENT-OFF* */
4632 VLIB_REGISTER_NODE (nat_default_node) = {
4633   .name = "nat-default",
4634   .vector_size = sizeof (u32),
4635   .format_trace = 0,
4636   .type = VLIB_NODE_TYPE_INTERNAL,
4637   .n_errors = 0,
4638   .n_next_nodes = NAT_N_NEXT,
4639   .next_nodes = {
4640     [NAT_NEXT_DROP] = "error-drop",
4641     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4642     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4643     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4644     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
4645     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4646     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4647     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4648     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
4649     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
4650   },
4651 };
4652 /* *INDENT-ON* */
4653
4654 /*
4655  * fd.io coding-style-patch-verification: ON
4656  *
4657  * Local Variables:
4658  * eval: (c-set-style "gnu")
4659  * End:
4660  */