src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <vnet/fib/fib_table.h>
  35 #include <vnet/fib/ip4_fib.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 /* *INDENT-OFF* */
  42 /* Hook up input features */
  43 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  44   .arc_name = "ip4-unicast",
  45   .node_name = "nat-pre-in2out",
  46   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  47 };
  48 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  49   .arc_name = "ip4-unicast",
  50   .node_name = "nat-pre-out2in",
  51   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  52                                "ip4-dhcp-client-detect"),
  53 };
  54 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-in2out-worker-handoff",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-out2in-worker-handoff",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  63                                "ip4-dhcp-client-detect"),
  64 };
  65 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  66   .arc_name = "ip4-unicast",
  67   .node_name = "nat44-in2out",
  68   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  69 };
  70 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  71   .arc_name = "ip4-unicast",
  72   .node_name = "nat44-out2in",
  73   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  74                                "ip4-dhcp-client-detect"),
  75 };
  76 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  77   .arc_name = "ip4-unicast",
  78   .node_name = "nat44-classify",
  79   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  80 };
  81 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  82   .arc_name = "ip4-unicast",
  83   .node_name = "nat44-det-in2out",
  84   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  85 };
  86 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat44-det-out2in",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  90                                "ip4-dhcp-client-detect"),
  91 };
  92 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-det-classify",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  96 };
  97 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  98   .arc_name = "ip4-unicast",
  99   .node_name = "nat44-ed-in2out",
 100   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-ed-out2in",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 106                                "ip4-dhcp-client-detect"),
 107 };
 108 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-ed-classify",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-handoff-classify",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 117 };
 118 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-in2out-fast",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-out2in-fast",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 127                                "ip4-dhcp-client-detect"),
 128 };
 129 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 130   .arc_name = "ip4-unicast",
 131   .node_name = "nat44-hairpin-dst",
 132   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 133 };
 134 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 135   .arc_name = "ip4-unicast",
 136   .node_name = "nat44-ed-hairpin-dst",
 137   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 138 };
 139
 140 /* Hook up output features */
 141 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 142   .arc_name = "ip4-output",
 143   .node_name = "nat44-in2out-output",
 144   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 145 };
 146 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 147   .arc_name = "ip4-output",
 148   .node_name = "nat44-in2out-output-worker-handoff",
 149   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 150 };
 151 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-hairpin-src",
 154   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 155 };
 156 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 157   .arc_name = "ip4-output",
 158   .node_name = "nat44-ed-in2out-output",
 159   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 162   .arc_name = "ip4-output",
 163   .node_name = "nat44-ed-hairpin-src",
 164   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 165 };
 166
 167 /* Hook up ip4-local features */
 168 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 169 {
 170   .arc_name = "ip4-local",
 171   .node_name = "nat44-hairpinning",
 172   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 173 };
 174 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 175 {
 176   .arc_name = "ip4-local",
 177   .node_name = "nat44-ed-hairpinning",
 178   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 179 };
 180
 181
 182 VLIB_PLUGIN_REGISTER () = {
 183     .version = VPP_BUILD_VER,
 184     .description = "Network Address Translation (NAT)",
 185 };
 186 /* *INDENT-ON* */
 187
 188 void
 189 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 190                        u8 is_ha)
 191 {
 192   snat_session_key_t key;
 193   clib_bihash_kv_8_8_t kv;
 194   nat_ed_ses_key_t ed_key;
 195   clib_bihash_kv_16_8_t ed_kv;
 196   snat_main_per_thread_data_t *tsm =
 197     vec_elt_at_index (sm->per_thread_data, thread_index);
 198
 199   if (is_fwd_bypass_session (s))
 200     {
 201       if (snat_is_unk_proto_session (s))
 202         {
 203           ed_key.proto = s->in2out.port;
 204           ed_key.r_port = 0;
 205           ed_key.l_port = 0;
 206         }
 207       else
 208         {
 209           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 210           ed_key.l_port = s->in2out.port;
 211           ed_key.r_port = s->ext_host_port;
 212         }
 213       ed_key.l_addr = s->in2out.addr;
 214       ed_key.r_addr = s->ext_host_addr;
 215       ed_key.fib_index = 0;
 216       ed_kv.key[0] = ed_key.as_u64[0];
 217       ed_kv.key[1] = ed_key.as_u64[1];
 218       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 219         nat_elog_warn ("in2out_ed key del failed");
 220       return;
 221     }
 222
 223   /* session lookup tables */
 224   if (is_ed_session (s))
 225     {
 226       if (is_affinity_sessions (s))
 227         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 228                              s->in2out.protocol, s->out2in.port);
 229       ed_key.l_addr = s->out2in.addr;
 230       ed_key.r_addr = s->ext_host_addr;
 231       ed_key.fib_index = s->out2in.fib_index;
 232       if (snat_is_unk_proto_session (s))
 233         {
 234           ed_key.proto = s->in2out.port;
 235           ed_key.r_port = 0;
 236           ed_key.l_port = 0;
 237         }
 238       else
 239         {
 240           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 241           ed_key.l_port = s->out2in.port;
 242           ed_key.r_port = s->ext_host_port;
 243         }
 244       ed_kv.key[0] = ed_key.as_u64[0];
 245       ed_kv.key[1] = ed_key.as_u64[1];
 246       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 247         nat_elog_warn ("out2in_ed key del failed");
 248       ed_key.l_addr = s->in2out.addr;
 249       ed_key.fib_index = s->in2out.fib_index;
 250       if (!snat_is_unk_proto_session (s))
 251         ed_key.l_port = s->in2out.port;
 252       if (is_twice_nat_session (s))
 253         {
 254           ed_key.r_addr = s->ext_host_nat_addr;
 255           ed_key.r_port = s->ext_host_nat_port;
 256         }
 257       ed_kv.key[0] = ed_key.as_u64[0];
 258       ed_kv.key[1] = ed_key.as_u64[1];
 259       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 260         nat_elog_warn ("in2out_ed key del failed");
 261
 262       if (!is_ha)
 263         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 264                                &s->in2out.addr, s->in2out.port,
 265                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 266                                &s->out2in.addr, s->out2in.port,
 267                                &s->ext_host_addr, s->ext_host_port,
 268                                s->in2out.protocol, is_twice_nat_session (s));
 269     }
 270   else
 271     {
 272       kv.key = s->in2out.as_u64;
 273       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 274         nat_elog_warn ("in2out key del failed");
 275       kv.key = s->out2in.as_u64;
 276       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 277         nat_elog_warn ("out2in key del failed");
 278
 279       if (!is_ha)
 280         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 281                                  &s->in2out.addr, s->in2out.port,
 282                                  &s->out2in.addr, s->out2in.port,
 283                                  s->in2out.protocol);
 284     }
 285
 286   if (snat_is_unk_proto_session (s))
 287     return;
 288
 289   if (!is_ha)
 290     {
 291       /* log NAT event */
 292       snat_ipfix_logging_nat44_ses_delete (thread_index,
 293                                            s->in2out.addr.as_u32,
 294                                            s->out2in.addr.as_u32,
 295                                            s->in2out.protocol,
 296                                            s->in2out.port,
 297                                            s->out2in.port,
 298                                            s->in2out.fib_index);
 299
 300       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 301                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 302                    thread_index);
 303     }
 304
 305   /* Twice NAT address and port for external host */
 306   if (is_twice_nat_session (s))
 307     {
 308       key.protocol = s->in2out.protocol;
 309       key.port = s->ext_host_nat_port;
 310       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 311       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 312                                           thread_index, &key);
 313     }
 314
 315   if (snat_is_session_static (s))
 316     return;
 317
 318   snat_free_outside_address_and_port (sm->addresses, thread_index,
 319                                       &s->out2in);
 320 }
 321
 322 snat_user_t *
 323 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 324                         u32 thread_index)
 325 {
 326   snat_user_t *u = 0;
 327   snat_user_key_t user_key;
 328   clib_bihash_kv_8_8_t kv, value;
 329   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 330   dlist_elt_t *per_user_list_head_elt;
 331
 332   user_key.addr.as_u32 = addr->as_u32;
 333   user_key.fib_index = fib_index;
 334   kv.key = user_key.as_u64;
 335
 336   /* Ever heard of the "user" = src ip4 address before? */
 337   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 338     {
 339       /* no, make a new one */
 340       pool_get (tsm->users, u);
 341       clib_memset (u, 0, sizeof (*u));
 342       u->addr.as_u32 = addr->as_u32;
 343       u->fib_index = fib_index;
 344
 345       pool_get (tsm->list_pool, per_user_list_head_elt);
 346
 347       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 348         tsm->list_pool;
 349
 350       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 351
 352       kv.value = u - tsm->users;
 353
 354       /* add user */
 355       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 356         nat_elog_warn ("user_hash keay add failed");
 357
 358       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 359                                pool_elts (tsm->users));
 360     }
 361   else
 362     {
 363       u = pool_elt_at_index (tsm->users, value.value);
 364     }
 365
 366   return u;
 367 }
 368
 369 snat_session_t *
 370 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 371                               u32 thread_index, f64 now)
 372 {
 373   snat_session_t *s;
 374   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 375   u32 oldest_per_user_translation_list_index, session_index;
 376   dlist_elt_t *oldest_per_user_translation_list_elt;
 377   dlist_elt_t *per_user_translation_list_elt;
 378
 379   /* Over quota? Recycle the least recently used translation */
 380   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 381     {
 382       oldest_per_user_translation_list_index =
 383         clib_dlist_remove_head (tsm->list_pool,
 384                                 u->sessions_per_user_list_head_index);
 385
 386       ASSERT (oldest_per_user_translation_list_index != ~0);
 387
 388       /* Add it back to the end of the LRU list */
 389       clib_dlist_addtail (tsm->list_pool,
 390                           u->sessions_per_user_list_head_index,
 391                           oldest_per_user_translation_list_index);
 392       /* Get the list element */
 393       oldest_per_user_translation_list_elt =
 394         pool_elt_at_index (tsm->list_pool,
 395                            oldest_per_user_translation_list_index);
 396
 397       /* Get the session index from the list element */
 398       session_index = oldest_per_user_translation_list_elt->value;
 399
 400       /* Get the session */
 401       s = pool_elt_at_index (tsm->sessions, session_index);
 402       nat_free_session_data (sm, s, thread_index, 0);
 403       if (snat_is_session_static (s))
 404         u->nstaticsessions--;
 405       else
 406         u->nsessions--;
 407       s->flags = 0;
 408       s->total_bytes = 0;
 409       s->total_pkts = 0;
 410       s->state = 0;
 411       s->ext_host_addr.as_u32 = 0;
 412       s->ext_host_port = 0;
 413       s->ext_host_nat_addr.as_u32 = 0;
 414       s->ext_host_nat_port = 0;
 415     }
 416   else
 417     {
 418       pool_get (tsm->sessions, s);
 419       clib_memset (s, 0, sizeof (*s));
 420
 421       /* Create list elts */
 422       pool_get (tsm->list_pool, per_user_translation_list_elt);
 423       clib_dlist_init (tsm->list_pool,
 424                        per_user_translation_list_elt - tsm->list_pool);
 425
 426       per_user_translation_list_elt->value = s - tsm->sessions;
 427       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 428       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 429
 430       clib_dlist_addtail (tsm->list_pool,
 431                           s->per_user_list_head_index,
 432                           per_user_translation_list_elt - tsm->list_pool);
 433
 434       s->user_index = u - tsm->users;
 435       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 436                                pool_elts (tsm->sessions));
 437     }
 438
 439   s->ha_last_refreshed = now;
 440
 441   return s;
 442 }
 443
 444 snat_session_t *
 445 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 446                       f64 now)
 447 {
 448   snat_session_t *s;
 449   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 450   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 451   u32 oldest_index;
 452   u64 sess_timeout_time;
 453
 454   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 455     goto alloc_new;
 456
 457   oldest_index =
 458     clib_dlist_remove_head (tsm->list_pool,
 459                             u->sessions_per_user_list_head_index);
 460   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 461   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 462   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 463   if (now >= sess_timeout_time)
 464     {
 465       clib_dlist_addtail (tsm->list_pool,
 466                           u->sessions_per_user_list_head_index, oldest_index);
 467       nat_free_session_data (sm, s, thread_index, 0);
 468       if (snat_is_session_static (s))
 469         u->nstaticsessions--;
 470       else
 471         u->nsessions--;
 472       s->flags = 0;
 473       s->total_bytes = 0;
 474       s->total_pkts = 0;
 475       s->state = 0;
 476       s->ext_host_addr.as_u32 = 0;
 477       s->ext_host_port = 0;
 478       s->ext_host_nat_addr.as_u32 = 0;
 479       s->ext_host_nat_port = 0;
 480     }
 481   else
 482     {
 483       clib_dlist_addhead (tsm->list_pool,
 484                           u->sessions_per_user_list_head_index, oldest_index);
 485       if ((u->nsessions + u->nstaticsessions) >=
 486           sm->max_translations_per_user)
 487         {
 488           nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 489                          clib_net_to_host_u32 (u->addr.as_u32));
 490           snat_ipfix_logging_max_entries_per_user
 491             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 492           return 0;
 493         }
 494       else
 495         {
 496         alloc_new:
 497           pool_get (tsm->sessions, s);
 498           clib_memset (s, 0, sizeof (*s));
 499
 500           /* Create list elts */
 501           pool_get (tsm->list_pool, per_user_translation_list_elt);
 502           clib_dlist_init (tsm->list_pool,
 503                            per_user_translation_list_elt - tsm->list_pool);
 504
 505           per_user_translation_list_elt->value = s - tsm->sessions;
 506           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 507           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 508
 509           clib_dlist_addtail (tsm->list_pool,
 510                               s->per_user_list_head_index,
 511                               per_user_translation_list_elt - tsm->list_pool);
 512         }
 513
 514       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 515                                pool_elts (tsm->sessions));
 516     }
 517
 518   s->ha_last_refreshed = now;
 519
 520   return s;
 521 }
 522
 523 void
 524 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 525                           int is_add)
 526 {
 527   fib_prefix_t prefix = {
 528     .fp_len = p_len,
 529     .fp_proto = FIB_PROTOCOL_IP4,
 530     .fp_addr = {
 531                 .ip4.as_u32 = addr->as_u32,
 532                 },
 533   };
 534   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 535
 536   if (is_add)
 537     fib_table_entry_update_one_path (fib_index,
 538                                      &prefix,
 539                                      FIB_SOURCE_PLUGIN_LOW,
 540                                      (FIB_ENTRY_FLAG_CONNECTED |
 541                                       FIB_ENTRY_FLAG_LOCAL |
 542                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 543                                      DPO_PROTO_IP4,
 544                                      NULL,
 545                                      sw_if_index,
 546                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 547   else
 548     fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
 549 }
 550
 551 int
 552 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 553                   u8 twice_nat)
 554 {
 555   snat_address_t *ap;
 556   snat_interface_t *i;
 557   vlib_thread_main_t *tm = vlib_get_thread_main ();
 558
 559   if (twice_nat && !sm->endpoint_dependent)
 560     return VNET_API_ERROR_FEATURE_DISABLED;
 561
 562   /* Check if address already exists */
 563   /* *INDENT-OFF* */
 564   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 565     {
 566       if (ap->addr.as_u32 == addr->as_u32)
 567         return VNET_API_ERROR_VALUE_EXIST;
 568     }
 569   /* *INDENT-ON* */
 570
 571   if (twice_nat)
 572     vec_add2 (sm->twice_nat_addresses, ap, 1);
 573   else
 574     vec_add2 (sm->addresses, ap, 1);
 575
 576   ap->addr = *addr;
 577   if (vrf_id != ~0)
 578     ap->fib_index =
 579       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 580                                          FIB_SOURCE_PLUGIN_LOW);
 581   else
 582     ap->fib_index = ~0;
 583 #define _(N, i, n, s) \
 584   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 585   ap->busy_##n##_ports = 0; \
 586   ap->busy_##n##_ports_per_thread = 0;\
 587   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 588   foreach_snat_protocol
 589 #undef _
 590     if (twice_nat)
 591     return 0;
 592
 593   /* Add external address to FIB */
 594   /* *INDENT-OFF* */
 595   pool_foreach (i, sm->interfaces,
 596   ({
 597     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 598       continue;
 599
 600     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 601     break;
 602   }));
 603   pool_foreach (i, sm->output_feature_interfaces,
 604   ({
 605     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 606       continue;
 607
 608     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 609     break;
 610   }));
 611   /* *INDENT-ON* */
 612
 613   return 0;
 614 }
 615
 616 static int
 617 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 618 {
 619   snat_static_mapping_t *m;
 620   /* *INDENT-OFF* */
 621   pool_foreach (m, sm->static_mappings,
 622   ({
 623       if (is_addr_only_static_mapping (m) ||
 624           is_out2in_only_static_mapping (m) ||
 625           is_identity_static_mapping (m))
 626         continue;
 627       if (m->external_addr.as_u32 == addr.as_u32)
 628         return 1;
 629   }));
 630   /* *INDENT-ON* */
 631
 632   return 0;
 633 }
 634
 635 void
 636 increment_v4_address (ip4_address_t * a)
 637 {
 638   u32 v;
 639
 640   v = clib_net_to_host_u32 (a->as_u32) + 1;
 641   a->as_u32 = clib_host_to_net_u32 (v);
 642 }
 643
 644 static void
 645 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 646                                        ip4_address_t l_addr,
 647                                        u16 l_port,
 648                                        u32 sw_if_index,
 649                                        u16 e_port,
 650                                        u32 vrf_id,
 651                                        snat_protocol_t proto,
 652                                        int addr_only, int is_add, u8 * tag,
 653                                        int twice_nat, int out2in_only,
 654                                        int identity_nat)
 655 {
 656   snat_static_map_resolve_t *rp;
 657
 658   vec_add2 (sm->to_resolve, rp, 1);
 659   rp->l_addr.as_u32 = l_addr.as_u32;
 660   rp->l_port = l_port;
 661   rp->sw_if_index = sw_if_index;
 662   rp->e_port = e_port;
 663   rp->vrf_id = vrf_id;
 664   rp->proto = proto;
 665   rp->addr_only = addr_only;
 666   rp->is_add = is_add;
 667   rp->twice_nat = twice_nat;
 668   rp->out2in_only = out2in_only;
 669   rp->identity_nat = identity_nat;
 670   rp->tag = vec_dup (tag);
 671 }
 672
 673 static u32
 674 get_thread_idx_by_port (u16 e_port)
 675 {
 676   snat_main_t *sm = &snat_main;
 677   u32 thread_idx = sm->num_workers;
 678   if (sm->num_workers > 1)
 679     {
 680       thread_idx =
 681         sm->first_worker_index +
 682         sm->workers[(e_port - 1024) / sm->port_per_thread];
 683     }
 684   return thread_idx;
 685 }
 686
 687 int
 688 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 689                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 690                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 691                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 692                          u8 identity_nat)
 693 {
 694   snat_main_t *sm = &snat_main;
 695   snat_static_mapping_t *m;
 696   snat_session_key_t m_key;
 697   clib_bihash_kv_8_8_t kv, value;
 698   snat_address_t *a = 0;
 699   u32 fib_index = ~0;
 700   snat_interface_t *interface;
 701   int i;
 702   snat_main_per_thread_data_t *tsm;
 703   snat_user_key_t u_key;
 704   snat_user_t *u;
 705   dlist_elt_t *head, *elt;
 706   u32 elt_index, head_index;
 707   u32 ses_index;
 708   u64 user_index;
 709   snat_session_t *s;
 710   snat_static_map_resolve_t *rp, *rp_match = 0;
 711   nat44_lb_addr_port_t *local;
 712   u32 find = ~0;
 713
 714   if (!sm->endpoint_dependent)
 715     {
 716       if (twice_nat || out2in_only)
 717         return VNET_API_ERROR_FEATURE_DISABLED;
 718     }
 719
 720   /* If the external address is a specific interface address */
 721   if (sw_if_index != ~0)
 722     {
 723       ip4_address_t *first_int_addr;
 724
 725       for (i = 0; i < vec_len (sm->to_resolve); i++)
 726         {
 727           rp = sm->to_resolve + i;
 728           if (rp->sw_if_index != sw_if_index ||
 729               rp->l_addr.as_u32 != l_addr.as_u32 ||
 730               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 731             continue;
 732
 733           if (!addr_only)
 734             {
 735               if ((rp->l_port != l_port && rp->e_port != e_port)
 736                   || rp->proto != proto)
 737                 continue;
 738             }
 739
 740           rp_match = rp;
 741           break;
 742         }
 743
 744       /* Might be already set... */
 745       first_int_addr = ip4_interface_first_address
 746         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 747
 748       if (is_add)
 749         {
 750           if (rp_match)
 751             return VNET_API_ERROR_VALUE_EXIST;
 752
 753           snat_add_static_mapping_when_resolved
 754             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 755              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 756
 757           /* DHCP resolution required? */
 758           if (first_int_addr == 0)
 759             {
 760               return 0;
 761             }
 762           else
 763             {
 764               e_addr.as_u32 = first_int_addr->as_u32;
 765               /* Identity mapping? */
 766               if (l_addr.as_u32 == 0)
 767                 l_addr.as_u32 = e_addr.as_u32;
 768             }
 769         }
 770       else
 771         {
 772           if (!rp_match)
 773             return VNET_API_ERROR_NO_SUCH_ENTRY;
 774
 775           vec_del1 (sm->to_resolve, i);
 776
 777           if (first_int_addr)
 778             {
 779               e_addr.as_u32 = first_int_addr->as_u32;
 780               /* Identity mapping? */
 781               if (l_addr.as_u32 == 0)
 782                 l_addr.as_u32 = e_addr.as_u32;
 783             }
 784           else
 785             return 0;
 786         }
 787     }
 788
 789   m_key.addr = e_addr;
 790   m_key.port = addr_only ? 0 : e_port;
 791   m_key.protocol = addr_only ? 0 : proto;
 792   m_key.fib_index = 0;
 793   kv.key = m_key.as_u64;
 794   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 795     m = 0;
 796   else
 797     m = pool_elt_at_index (sm->static_mappings, value.value);
 798
 799   if (is_add)
 800     {
 801       if (m)
 802         {
 803           if (is_identity_static_mapping (m))
 804             {
 805               /* *INDENT-OFF* */
 806               pool_foreach (local, m->locals,
 807               ({
 808                 if (local->vrf_id == vrf_id)
 809                   return VNET_API_ERROR_VALUE_EXIST;
 810               }));
 811               /* *INDENT-ON* */
 812               pool_get (m->locals, local);
 813               local->vrf_id = vrf_id;
 814               local->fib_index =
 815                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 816                                                    FIB_SOURCE_PLUGIN_LOW);
 817               m_key.addr = m->local_addr;
 818               m_key.port = m->local_port;
 819               m_key.protocol = m->proto;
 820               m_key.fib_index = local->fib_index;
 821               kv.key = m_key.as_u64;
 822               kv.value = m - sm->static_mappings;
 823               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 824               return 0;
 825             }
 826           else
 827             return VNET_API_ERROR_VALUE_EXIST;
 828         }
 829
 830       if (twice_nat && addr_only)
 831         return VNET_API_ERROR_UNSUPPORTED;
 832
 833       /* Convert VRF id to FIB index */
 834       if (vrf_id != ~0)
 835         fib_index =
 836           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 837                                              FIB_SOURCE_PLUGIN_LOW);
 838       /* If not specified use inside VRF id from SNAT plugin startup config */
 839       else
 840         {
 841           fib_index = sm->inside_fib_index;
 842           vrf_id = sm->inside_vrf_id;
 843           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
 844         }
 845
 846       if (!(out2in_only || identity_nat))
 847         {
 848           m_key.addr = l_addr;
 849           m_key.port = addr_only ? 0 : l_port;
 850           m_key.protocol = addr_only ? 0 : proto;
 851           m_key.fib_index = fib_index;
 852           kv.key = m_key.as_u64;
 853           if (!clib_bihash_search_8_8
 854               (&sm->static_mapping_by_local, &kv, &value))
 855             return VNET_API_ERROR_VALUE_EXIST;
 856         }
 857
 858       /* Find external address in allocated addresses and reserve port for
 859          address and port pair mapping when dynamic translations enabled */
 860       if (!(addr_only || sm->static_mapping_only || out2in_only))
 861         {
 862           for (i = 0; i < vec_len (sm->addresses); i++)
 863             {
 864               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 865                 {
 866                   a = sm->addresses + i;
 867                   /* External port must be unused */
 868                   switch (proto)
 869                     {
 870 #define _(N, j, n, s) \
 871                     case SNAT_PROTOCOL_##N: \
 872                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 873                         return VNET_API_ERROR_INVALID_VALUE; \
 874                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 875                       if (e_port > 1024) \
 876                         { \
 877                           a->busy_##n##_ports++; \
 878                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 879                         } \
 880                       break;
 881                       foreach_snat_protocol
 882 #undef _
 883                     default:
 884                       nat_elog_info ("unknown protocol");
 885                       return VNET_API_ERROR_INVALID_VALUE_2;
 886                     }
 887                   break;
 888                 }
 889             }
 890           /* External address must be allocated */
 891           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 892             {
 893               if (sw_if_index != ~0)
 894                 {
 895                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 896                     {
 897                       rp = sm->to_resolve + i;
 898                       if (rp->addr_only)
 899                         continue;
 900                       if (rp->sw_if_index != sw_if_index &&
 901                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 902                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 903                           rp->e_port != e_port && rp->proto != proto)
 904                         continue;
 905
 906                       vec_del1 (sm->to_resolve, i);
 907                       break;
 908                     }
 909                 }
 910               return VNET_API_ERROR_NO_SUCH_ENTRY;
 911             }
 912         }
 913
 914       pool_get (sm->static_mappings, m);
 915       clib_memset (m, 0, sizeof (*m));
 916       m->tag = vec_dup (tag);
 917       m->local_addr = l_addr;
 918       m->external_addr = e_addr;
 919       m->twice_nat = twice_nat;
 920       if (out2in_only)
 921         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 922       if (addr_only)
 923         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 924       if (identity_nat)
 925         {
 926           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 927           pool_get (m->locals, local);
 928           local->vrf_id = vrf_id;
 929           local->fib_index = fib_index;
 930         }
 931       else
 932         {
 933           m->vrf_id = vrf_id;
 934           m->fib_index = fib_index;
 935         }
 936       if (!addr_only)
 937         {
 938           m->local_port = l_port;
 939           m->external_port = e_port;
 940           m->proto = proto;
 941         }
 942
 943       if (sm->num_workers > 1)
 944         {
 945           ip4_header_t ip = {
 946             .src_address = m->local_addr,
 947           };
 948           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 949           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 950         }
 951       else
 952         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 953
 954       m_key.addr = m->local_addr;
 955       m_key.port = m->local_port;
 956       m_key.protocol = m->proto;
 957       m_key.fib_index = fib_index;
 958       kv.key = m_key.as_u64;
 959       kv.value = m - sm->static_mappings;
 960       if (!out2in_only)
 961         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 962
 963       m_key.addr = m->external_addr;
 964       m_key.port = m->external_port;
 965       m_key.fib_index = 0;
 966       kv.key = m_key.as_u64;
 967       kv.value = m - sm->static_mappings;
 968       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 969
 970       /* Delete dynamic sessions matching local address (+ local port) */
 971       if (!(sm->static_mapping_only))
 972         {
 973           u_key.addr = m->local_addr;
 974           u_key.fib_index = m->fib_index;
 975           kv.key = u_key.as_u64;
 976           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 977             {
 978               user_index = value.value;
 979               u = pool_elt_at_index (tsm->users, user_index);
 980               if (u->nsessions)
 981                 {
 982                   head_index = u->sessions_per_user_list_head_index;
 983                   head = pool_elt_at_index (tsm->list_pool, head_index);
 984                   elt_index = head->next;
 985                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 986                   ses_index = elt->value;
 987                   while (ses_index != ~0)
 988                     {
 989                       s = pool_elt_at_index (tsm->sessions, ses_index);
 990                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 991                       ses_index = elt->value;
 992
 993                       if (snat_is_session_static (s))
 994                         continue;
 995
 996                       if (!addr_only
 997                           && (clib_net_to_host_u16 (s->in2out.port) !=
 998                               m->local_port))
 999                         continue;
1000
1001                       nat_free_session_data (sm, s,
1002                                              tsm - sm->per_thread_data, 0);
1003                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1004
1005                       if (!addr_only && !sm->endpoint_dependent)
1006                         break;
1007                     }
1008                 }
1009             }
1010         }
1011     }
1012   else
1013     {
1014       if (!m)
1015         {
1016           if (sw_if_index != ~0)
1017             return 0;
1018           else
1019             return VNET_API_ERROR_NO_SUCH_ENTRY;
1020         }
1021
1022       if (identity_nat)
1023         {
1024           if (vrf_id == ~0)
1025             vrf_id = sm->inside_vrf_id;
1026
1027           /* *INDENT-OFF* */
1028           pool_foreach (local, m->locals,
1029           ({
1030             if (local->vrf_id == vrf_id)
1031               find = local - m->locals;
1032           }));
1033           /* *INDENT-ON* */
1034           if (find == ~0)
1035             return VNET_API_ERROR_NO_SUCH_ENTRY;
1036
1037           local = pool_elt_at_index (m->locals, find);
1038           fib_index = local->fib_index;
1039           pool_put (m->locals, local);
1040         }
1041       else
1042         fib_index = m->fib_index;
1043
1044       /* Free external address port */
1045       if (!(addr_only || sm->static_mapping_only || out2in_only))
1046         {
1047           for (i = 0; i < vec_len (sm->addresses); i++)
1048             {
1049               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1050                 {
1051                   a = sm->addresses + i;
1052                   switch (proto)
1053                     {
1054 #define _(N, j, n, s) \
1055                     case SNAT_PROTOCOL_##N: \
1056                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1057                       if (e_port > 1024) \
1058                         { \
1059                           a->busy_##n##_ports--; \
1060                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1061                         } \
1062                       break;
1063                       foreach_snat_protocol
1064 #undef _
1065                     default:
1066                       nat_elog_info ("unknown protocol");
1067                       return VNET_API_ERROR_INVALID_VALUE_2;
1068                     }
1069                   break;
1070                 }
1071             }
1072         }
1073
1074       if (sm->num_workers > 1)
1075         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1076       else
1077         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1078
1079       m_key.addr = m->local_addr;
1080       m_key.port = m->local_port;
1081       m_key.protocol = m->proto;
1082       m_key.fib_index = fib_index;
1083       kv.key = m_key.as_u64;
1084       if (!out2in_only)
1085         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1086
1087       /* Delete session(s) for static mapping if exist */
1088       if (!(sm->static_mapping_only) ||
1089           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1090         {
1091           u_key.addr = m->local_addr;
1092           u_key.fib_index = fib_index;
1093           kv.key = u_key.as_u64;
1094           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1095             {
1096               user_index = value.value;
1097               u = pool_elt_at_index (tsm->users, user_index);
1098               if (u->nstaticsessions)
1099                 {
1100                   head_index = u->sessions_per_user_list_head_index;
1101                   head = pool_elt_at_index (tsm->list_pool, head_index);
1102                   elt_index = head->next;
1103                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1104                   ses_index = elt->value;
1105                   while (ses_index != ~0)
1106                     {
1107                       s = pool_elt_at_index (tsm->sessions, ses_index);
1108                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1109                       ses_index = elt->value;
1110
1111                       if (!addr_only)
1112                         {
1113                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1114                               (clib_net_to_host_u16 (s->out2in.port) !=
1115                                e_port))
1116                             continue;
1117                         }
1118
1119                       if (is_lb_session (s))
1120                         continue;
1121
1122                       if (!snat_is_session_static (s))
1123                         continue;
1124
1125                       nat_free_session_data (sm, s,
1126                                              tsm - sm->per_thread_data, 0);
1127                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1128
1129                       if (!addr_only && !sm->endpoint_dependent)
1130                         break;
1131                     }
1132                 }
1133             }
1134         }
1135
1136       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1137       if (pool_elts (m->locals))
1138         return 0;
1139
1140       m_key.addr = m->external_addr;
1141       m_key.port = m->external_port;
1142       m_key.fib_index = 0;
1143       kv.key = m_key.as_u64;
1144       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1145
1146       vec_free (m->tag);
1147       vec_free (m->workers);
1148       /* Delete static mapping from pool */
1149       pool_put (sm->static_mappings, m);
1150     }
1151
1152   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1153     return 0;
1154
1155   /* Add/delete external address to FIB */
1156   /* *INDENT-OFF* */
1157   pool_foreach (interface, sm->interfaces,
1158   ({
1159     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1160       continue;
1161
1162     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1163     break;
1164   }));
1165   pool_foreach (interface, sm->output_feature_interfaces,
1166   ({
1167     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1168       continue;
1169
1170     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1171     break;
1172   }));
1173   /* *INDENT-ON* */
1174
1175   return 0;
1176 }
1177
1178 int
1179 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1180                                  snat_protocol_t proto,
1181                                  nat44_lb_addr_port_t * locals, u8 is_add,
1182                                  twice_nat_type_t twice_nat, u8 out2in_only,
1183                                  u8 * tag, u32 affinity)
1184 {
1185   snat_main_t *sm = &snat_main;
1186   snat_static_mapping_t *m;
1187   snat_session_key_t m_key;
1188   clib_bihash_kv_8_8_t kv, value;
1189   snat_address_t *a = 0;
1190   int i;
1191   nat44_lb_addr_port_t *local;
1192   u32 elt_index, head_index, ses_index;
1193   snat_main_per_thread_data_t *tsm;
1194   snat_user_key_t u_key;
1195   snat_user_t *u;
1196   snat_session_t *s;
1197   dlist_elt_t *head, *elt;
1198   uword *bitmap = 0;
1199
1200   if (!sm->endpoint_dependent)
1201     return VNET_API_ERROR_FEATURE_DISABLED;
1202
1203   m_key.addr = e_addr;
1204   m_key.port = e_port;
1205   m_key.protocol = proto;
1206   m_key.fib_index = 0;
1207   kv.key = m_key.as_u64;
1208   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1209     m = 0;
1210   else
1211     m = pool_elt_at_index (sm->static_mappings, value.value);
1212
1213   if (is_add)
1214     {
1215       if (m)
1216         return VNET_API_ERROR_VALUE_EXIST;
1217
1218       if (vec_len (locals) < 2)
1219         return VNET_API_ERROR_INVALID_VALUE;
1220
1221       /* Find external address in allocated addresses and reserve port for
1222          address and port pair mapping when dynamic translations enabled */
1223       if (!(sm->static_mapping_only || out2in_only))
1224         {
1225           for (i = 0; i < vec_len (sm->addresses); i++)
1226             {
1227               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1228                 {
1229                   a = sm->addresses + i;
1230                   /* External port must be unused */
1231                   switch (proto)
1232                     {
1233 #define _(N, j, n, s) \
1234                     case SNAT_PROTOCOL_##N: \
1235                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1236                         return VNET_API_ERROR_INVALID_VALUE; \
1237                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1238                       if (e_port > 1024) \
1239                         { \
1240                           a->busy_##n##_ports++; \
1241                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1242                         } \
1243                       break;
1244                       foreach_snat_protocol
1245 #undef _
1246                     default:
1247                       nat_elog_info ("unknown protocol");
1248                       return VNET_API_ERROR_INVALID_VALUE_2;
1249                     }
1250                   break;
1251                 }
1252             }
1253           /* External address must be allocated */
1254           if (!a)
1255             return VNET_API_ERROR_NO_SUCH_ENTRY;
1256         }
1257
1258       pool_get (sm->static_mappings, m);
1259       clib_memset (m, 0, sizeof (*m));
1260       m->tag = vec_dup (tag);
1261       m->external_addr = e_addr;
1262       m->external_port = e_port;
1263       m->proto = proto;
1264       m->twice_nat = twice_nat;
1265       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1266       if (out2in_only)
1267         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1268       m->affinity = affinity;
1269
1270       if (affinity)
1271         m->affinity_per_service_list_head_index =
1272           nat_affinity_get_per_service_list_head_index ();
1273       else
1274         m->affinity_per_service_list_head_index = ~0;
1275
1276       m_key.addr = m->external_addr;
1277       m_key.port = m->external_port;
1278       m_key.protocol = m->proto;
1279       m_key.fib_index = 0;
1280       kv.key = m_key.as_u64;
1281       kv.value = m - sm->static_mappings;
1282       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1283         {
1284           nat_elog_err ("static_mapping_by_external key add failed");
1285           return VNET_API_ERROR_UNSPECIFIED;
1286         }
1287
1288       m_key.fib_index = m->fib_index;
1289       for (i = 0; i < vec_len (locals); i++)
1290         {
1291           locals[i].fib_index =
1292             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1293                                                locals[i].vrf_id,
1294                                                FIB_SOURCE_PLUGIN_LOW);
1295           m_key.addr = locals[i].addr;
1296           m_key.fib_index = locals[i].fib_index;
1297           if (!out2in_only)
1298             {
1299               m_key.port = locals[i].port;
1300               kv.key = m_key.as_u64;
1301               kv.value = m - sm->static_mappings;
1302               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1303             }
1304           locals[i].prefix = (i == 0) ? locals[i].probability :
1305             (locals[i - 1].prefix + locals[i].probability);
1306           pool_get (m->locals, local);
1307           *local = locals[i];
1308           if (sm->num_workers > 1)
1309             {
1310               ip4_header_t ip = {
1311                 .src_address = locals[i].addr,
1312               };
1313               bitmap =
1314                 clib_bitmap_set (bitmap,
1315                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1316                                  1);
1317             }
1318         }
1319
1320       /* Assign workers */
1321       if (sm->num_workers > 1)
1322         {
1323           /* *INDENT-OFF* */
1324           clib_bitmap_foreach (i, bitmap,
1325             ({
1326                vec_add1(m->workers, i);
1327             }));
1328           /* *INDENT-ON* */
1329         }
1330     }
1331   else
1332     {
1333       if (!m)
1334         return VNET_API_ERROR_NO_SUCH_ENTRY;
1335
1336       if (!is_lb_static_mapping (m))
1337         return VNET_API_ERROR_INVALID_VALUE;
1338
1339       /* Free external address port */
1340       if (!(sm->static_mapping_only || out2in_only))
1341         {
1342           for (i = 0; i < vec_len (sm->addresses); i++)
1343             {
1344               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1345                 {
1346                   a = sm->addresses + i;
1347                   switch (proto)
1348                     {
1349 #define _(N, j, n, s) \
1350                     case SNAT_PROTOCOL_##N: \
1351                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1352                       if (e_port > 1024) \
1353                         { \
1354                           a->busy_##n##_ports--; \
1355                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1356                         } \
1357                       break;
1358                       foreach_snat_protocol
1359 #undef _
1360                     default:
1361                       nat_elog_info ("unknown protocol");
1362                       return VNET_API_ERROR_INVALID_VALUE_2;
1363                     }
1364                   break;
1365                 }
1366             }
1367         }
1368
1369       m_key.addr = m->external_addr;
1370       m_key.port = m->external_port;
1371       m_key.protocol = m->proto;
1372       m_key.fib_index = 0;
1373       kv.key = m_key.as_u64;
1374       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1375         {
1376           nat_elog_err ("static_mapping_by_external key del failed");
1377           return VNET_API_ERROR_UNSPECIFIED;
1378         }
1379
1380       /* *INDENT-OFF* */
1381       pool_foreach (local, m->locals,
1382       ({
1383           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1384                             FIB_SOURCE_PLUGIN_LOW);
1385           m_key.addr = local->addr;
1386           if (!out2in_only)
1387             {
1388               m_key.port = local->port;
1389               m_key.fib_index = local->fib_index;
1390               kv.key = m_key.as_u64;
1391               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1392                 {
1393                   nat_elog_err ("static_mapping_by_local key del failed");
1394                   return VNET_API_ERROR_UNSPECIFIED;
1395                 }
1396             }
1397
1398           if (sm->num_workers > 1)
1399             {
1400               ip4_header_t ip = {
1401                 .src_address = local->addr,
1402               };
1403               tsm = vec_elt_at_index (sm->per_thread_data,
1404                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1405             }
1406           else
1407             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1408
1409           /* Delete sessions */
1410           u_key.addr = local->addr;
1411           u_key.fib_index = local->fib_index;
1412           kv.key = u_key.as_u64;
1413           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1414             {
1415               u = pool_elt_at_index (tsm->users, value.value);
1416               if (u->nstaticsessions)
1417                 {
1418                   head_index = u->sessions_per_user_list_head_index;
1419                   head = pool_elt_at_index (tsm->list_pool, head_index);
1420                   elt_index = head->next;
1421                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1422                   ses_index = elt->value;
1423                   while (ses_index != ~0)
1424                     {
1425                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1426                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1427                       ses_index = elt->value;
1428
1429                       if (!(is_lb_session (s)))
1430                         continue;
1431
1432                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1433                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1434                         continue;
1435
1436                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1437                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1438                     }
1439                 }
1440             }
1441       }));
1442       /* *INDENT-ON* */
1443       if (m->affinity)
1444         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1445       pool_free (m->locals);
1446       vec_free (m->tag);
1447       vec_free (m->workers);
1448
1449       pool_put (sm->static_mappings, m);
1450     }
1451
1452   return 0;
1453 }
1454
1455 int
1456 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1457                                        ip4_address_t l_addr, u16 l_port,
1458                                        snat_protocol_t proto, u32 vrf_id,
1459                                        u8 probability, u8 is_add)
1460 {
1461   snat_main_t *sm = &snat_main;
1462   snat_static_mapping_t *m = 0;
1463   snat_session_key_t m_key;
1464   clib_bihash_kv_8_8_t kv, value;
1465   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1466   snat_main_per_thread_data_t *tsm;
1467   snat_user_key_t u_key;
1468   snat_user_t *u;
1469   snat_session_t *s;
1470   dlist_elt_t *head, *elt;
1471   u32 elt_index, head_index, ses_index, *locals = 0;
1472   uword *bitmap = 0;
1473   int i;
1474
1475   if (!sm->endpoint_dependent)
1476     return VNET_API_ERROR_FEATURE_DISABLED;
1477
1478   m_key.addr = e_addr;
1479   m_key.port = e_port;
1480   m_key.protocol = proto;
1481   m_key.fib_index = 0;
1482   kv.key = m_key.as_u64;
1483   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1484     m = pool_elt_at_index (sm->static_mappings, value.value);
1485
1486   if (!m)
1487     return VNET_API_ERROR_NO_SUCH_ENTRY;
1488
1489   if (!is_lb_static_mapping (m))
1490     return VNET_API_ERROR_INVALID_VALUE;
1491
1492   /* *INDENT-OFF* */
1493   pool_foreach (local, m->locals,
1494   ({
1495     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1496         (local->vrf_id == vrf_id))
1497       {
1498         match_local = local;
1499         break;
1500       }
1501   }));
1502   /* *INDENT-ON* */
1503
1504   if (is_add)
1505     {
1506       if (match_local)
1507         return VNET_API_ERROR_VALUE_EXIST;
1508
1509       pool_get (m->locals, local);
1510       clib_memset (local, 0, sizeof (*local));
1511       local->addr.as_u32 = l_addr.as_u32;
1512       local->port = l_port;
1513       local->probability = probability;
1514       local->vrf_id = vrf_id;
1515       local->fib_index =
1516         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1517                                            FIB_SOURCE_PLUGIN_LOW);
1518
1519       if (!is_out2in_only_static_mapping (m))
1520         {
1521           m_key.addr = l_addr;
1522           m_key.port = l_port;
1523           m_key.fib_index = local->fib_index;
1524           kv.key = m_key.as_u64;
1525           kv.value = m - sm->static_mappings;
1526           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1527             nat_elog_err ("static_mapping_by_local key add failed");
1528         }
1529     }
1530   else
1531     {
1532       if (!match_local)
1533         return VNET_API_ERROR_NO_SUCH_ENTRY;
1534
1535       if (pool_elts (m->locals) < 3)
1536         return VNET_API_ERROR_UNSPECIFIED;
1537
1538       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1539                         FIB_SOURCE_PLUGIN_LOW);
1540
1541       if (!is_out2in_only_static_mapping (m))
1542         {
1543           m_key.addr = l_addr;
1544           m_key.port = l_port;
1545           m_key.fib_index = match_local->fib_index;
1546           kv.key = m_key.as_u64;
1547           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1548             nat_elog_err ("static_mapping_by_local key del failed");
1549         }
1550
1551       if (sm->num_workers > 1)
1552         {
1553           ip4_header_t ip = {
1554             .src_address = local->addr,
1555           };
1556           tsm = vec_elt_at_index (sm->per_thread_data,
1557                                   sm->worker_in2out_cb (&ip, m->fib_index,
1558                                                         0));
1559         }
1560       else
1561         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1562
1563       /* Delete sessions */
1564       u_key.addr = match_local->addr;
1565       u_key.fib_index = match_local->fib_index;
1566       kv.key = u_key.as_u64;
1567       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1568         {
1569           u = pool_elt_at_index (tsm->users, value.value);
1570           if (u->nstaticsessions)
1571             {
1572               head_index = u->sessions_per_user_list_head_index;
1573               head = pool_elt_at_index (tsm->list_pool, head_index);
1574               elt_index = head->next;
1575               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1576               ses_index = elt->value;
1577               while (ses_index != ~0)
1578                 {
1579                   s = pool_elt_at_index (tsm->sessions, ses_index);
1580                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1581                   ses_index = elt->value;
1582
1583                   if (!(is_lb_session (s)))
1584                     continue;
1585
1586                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1587                       (clib_net_to_host_u16 (s->in2out.port) !=
1588                        match_local->port))
1589                     continue;
1590
1591                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1592                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1593                 }
1594             }
1595         }
1596
1597       pool_put (m->locals, match_local);
1598     }
1599
1600   vec_free (m->workers);
1601
1602   /* *INDENT-OFF* */
1603   pool_foreach (local, m->locals,
1604   ({
1605     vec_add1 (locals, local - m->locals);
1606     if (sm->num_workers > 1)
1607       {
1608         ip4_header_t ip;
1609         ip.src_address.as_u32 = local->addr.as_u32,
1610         bitmap = clib_bitmap_set (bitmap,
1611                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1612                                   1);
1613       }
1614   }));
1615   /* *INDENT-ON* */
1616
1617   ASSERT (vec_len (locals) > 1);
1618
1619   local = pool_elt_at_index (m->locals, locals[0]);
1620   local->prefix = local->probability;
1621   for (i = 1; i < vec_len (locals); i++)
1622     {
1623       local = pool_elt_at_index (m->locals, locals[i]);
1624       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1625       local->prefix = local->probability + prev_local->prefix;
1626     }
1627
1628   /* Assign workers */
1629   if (sm->num_workers > 1)
1630     {
1631       /* *INDENT-OFF* */
1632       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1633       /* *INDENT-ON* */
1634     }
1635
1636   return 0;
1637 }
1638
1639 int
1640 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1641                   u8 twice_nat)
1642 {
1643   snat_address_t *a = 0;
1644   snat_session_t *ses;
1645   u32 *ses_to_be_removed = 0, *ses_index;
1646   snat_main_per_thread_data_t *tsm;
1647   snat_static_mapping_t *m;
1648   snat_interface_t *interface;
1649   int i;
1650   snat_address_t *addresses =
1651     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1652
1653   /* Find SNAT address */
1654   for (i = 0; i < vec_len (addresses); i++)
1655     {
1656       if (addresses[i].addr.as_u32 == addr.as_u32)
1657         {
1658           a = addresses + i;
1659           break;
1660         }
1661     }
1662   if (!a)
1663     return VNET_API_ERROR_NO_SUCH_ENTRY;
1664
1665   if (delete_sm)
1666     {
1667       /* *INDENT-OFF* */
1668       pool_foreach (m, sm->static_mappings,
1669       ({
1670           if (m->external_addr.as_u32 == addr.as_u32)
1671             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1672                                             m->local_port, m->external_port,
1673                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1674                                             m->proto, 0, m->twice_nat,
1675                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1676       }));
1677       /* *INDENT-ON* */
1678     }
1679   else
1680     {
1681       /* Check if address is used in some static mapping */
1682       if (is_snat_address_used_in_static_mapping (sm, addr))
1683         {
1684           nat_elog_notice ("address used in static mapping");
1685           return VNET_API_ERROR_UNSPECIFIED;
1686         }
1687     }
1688
1689   if (a->fib_index != ~0)
1690     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1691
1692   /* Delete sessions using address */
1693   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1694     {
1695       /* *INDENT-OFF* */
1696       vec_foreach (tsm, sm->per_thread_data)
1697         {
1698           pool_foreach (ses, tsm->sessions, ({
1699             if (ses->out2in.addr.as_u32 == addr.as_u32)
1700               {
1701                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1702                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1703               }
1704           }));
1705
1706           vec_foreach (ses_index, ses_to_be_removed)
1707             {
1708               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1709               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1710             }
1711
1712           vec_free (ses_to_be_removed);
1713         }
1714       /* *INDENT-ON* */
1715     }
1716
1717 #define _(N, i, n, s) \
1718   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1719   vec_free (a->busy_##n##_ports_per_thread);
1720   foreach_snat_protocol
1721 #undef _
1722     if (twice_nat)
1723     {
1724       vec_del1 (sm->twice_nat_addresses, i);
1725       return 0;
1726     }
1727   else
1728     vec_del1 (sm->addresses, i);
1729
1730   /* Delete external address from FIB */
1731   /* *INDENT-OFF* */
1732   pool_foreach (interface, sm->interfaces,
1733   ({
1734     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1735       continue;
1736
1737     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1738     break;
1739   }));
1740   pool_foreach (interface, sm->output_feature_interfaces,
1741   ({
1742     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1743       continue;
1744
1745     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1746     break;
1747   }));
1748   /* *INDENT-ON* */
1749
1750   return 0;
1751 }
1752
1753 int
1754 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1755 {
1756   snat_main_t *sm = &snat_main;
1757   snat_interface_t *i;
1758   const char *feature_name, *del_feature_name;
1759   snat_address_t *ap;
1760   snat_static_mapping_t *m;
1761   snat_det_map_t *dm;
1762   nat_outside_fib_t *outside_fib;
1763   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1764                                                        sw_if_index);
1765
1766   if (sm->out2in_dpo && !is_inside)
1767     return VNET_API_ERROR_UNSUPPORTED;
1768
1769   /* *INDENT-OFF* */
1770   pool_foreach (i, sm->output_feature_interfaces,
1771   ({
1772     if (i->sw_if_index == sw_if_index)
1773       return VNET_API_ERROR_VALUE_EXIST;
1774   }));
1775   /* *INDENT-ON* */
1776
1777   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1778     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1779   else
1780     {
1781       if (sm->num_workers > 1 && !sm->deterministic)
1782         feature_name =
1783           is_inside ? "nat44-in2out-worker-handoff" :
1784           "nat44-out2in-worker-handoff";
1785       else if (sm->deterministic)
1786         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1787       else if (sm->endpoint_dependent)
1788         {
1789           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1790         }
1791       else
1792         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1793     }
1794
1795   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1796     sm->fq_in2out_index =
1797       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1798
1799   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1800     sm->fq_out2in_index =
1801       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1802
1803   if (!is_inside)
1804     {
1805       /* *INDENT-OFF* */
1806       vec_foreach (outside_fib, sm->outside_fibs)
1807         {
1808           if (outside_fib->fib_index == fib_index)
1809             {
1810               if (is_del)
1811                 {
1812                   outside_fib->refcount--;
1813                   if (!outside_fib->refcount)
1814                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1815                 }
1816               else
1817                 outside_fib->refcount++;
1818               goto feature_set;
1819             }
1820         }
1821       /* *INDENT-ON* */
1822       if (!is_del)
1823         {
1824           vec_add2 (sm->outside_fibs, outside_fib, 1);
1825           outside_fib->refcount = 1;
1826           outside_fib->fib_index = fib_index;
1827         }
1828     }
1829 feature_set:
1830   /* *INDENT-OFF* */
1831   pool_foreach (i, sm->interfaces,
1832   ({
1833     if (i->sw_if_index == sw_if_index)
1834       {
1835         if (is_del)
1836           {
1837             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1838               {
1839                 if (is_inside)
1840                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1841                 else
1842                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1843
1844                 if (sm->num_workers > 1 && !sm->deterministic)
1845                   {
1846                     del_feature_name = "nat44-handoff-classify";
1847                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1848                                                  "nat44-out2in-worker-handoff";
1849                   }
1850                 else if (sm->deterministic)
1851                   {
1852                     del_feature_name = "nat44-det-classify";
1853                     feature_name = !is_inside ?  "nat44-det-in2out" :
1854                                                  "nat44-det-out2in";
1855                   }
1856                 else if (sm->endpoint_dependent)
1857                   {
1858                     del_feature_name = "nat44-ed-classify";
1859                     feature_name = !is_inside ?  "nat-pre-in2out" :
1860                                                  "nat-pre-out2in";
1861                   }
1862                 else
1863                   {
1864                     del_feature_name = "nat44-classify";
1865                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1866                   }
1867
1868                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1869                                              sw_if_index, 0, 0, 0);
1870                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1871                                              sw_if_index, 1, 0, 0);
1872                 if (!is_inside)
1873                   {
1874                     if (sm->endpoint_dependent)
1875                       vnet_feature_enable_disable ("ip4-local",
1876                                                    "nat44-ed-hairpinning",
1877                                                    sw_if_index, 1, 0, 0);
1878                     else if (!sm->deterministic)
1879                       vnet_feature_enable_disable ("ip4-local",
1880                                                    "nat44-hairpinning",
1881                                                    sw_if_index, 1, 0, 0);
1882                   }
1883               }
1884             else
1885               {
1886                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1887                                              sw_if_index, 0, 0, 0);
1888                 pool_put (sm->interfaces, i);
1889                 if (is_inside)
1890                   {
1891                     if (sm->endpoint_dependent)
1892                       vnet_feature_enable_disable ("ip4-local",
1893                                                    "nat44-ed-hairpinning",
1894                                                    sw_if_index, 0, 0, 0);
1895                     else if (!sm->deterministic)
1896                       vnet_feature_enable_disable ("ip4-local",
1897                                                    "nat44-hairpinning",
1898                                                    sw_if_index, 0, 0, 0);
1899                   }
1900               }
1901           }
1902         else
1903           {
1904             if ((nat_interface_is_inside(i) && is_inside) ||
1905                 (nat_interface_is_outside(i) && !is_inside))
1906               return 0;
1907
1908             if (sm->num_workers > 1 && !sm->deterministic)
1909               {
1910                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1911                                                  "nat44-out2in-worker-handoff";
1912                 feature_name = "nat44-handoff-classify";
1913               }
1914             else if (sm->deterministic)
1915               {
1916                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1917                                                  "nat44-det-out2in";
1918                 feature_name = "nat44-det-classify";
1919               }
1920             else if (sm->endpoint_dependent)
1921               {
1922                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
1923                                                  "nat-pre-out2in";
1924
1925                 feature_name = "nat44-ed-classify";
1926               }
1927             else
1928               {
1929                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1930                 feature_name = "nat44-classify";
1931               }
1932
1933             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1934                                          sw_if_index, 0, 0, 0);
1935             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1936                                          sw_if_index, 1, 0, 0);
1937             if (!is_inside)
1938               {
1939                 if (sm->endpoint_dependent)
1940                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1941                                                sw_if_index, 0, 0, 0);
1942                 else if (!sm->deterministic)
1943                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1944                                                sw_if_index, 0, 0, 0);
1945               }
1946             goto set_flags;
1947           }
1948
1949         goto fib;
1950       }
1951   }));
1952   /* *INDENT-ON* */
1953
1954   if (is_del)
1955     return VNET_API_ERROR_NO_SUCH_ENTRY;
1956
1957   pool_get (sm->interfaces, i);
1958   i->sw_if_index = sw_if_index;
1959   i->flags = 0;
1960   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1961                                0);
1962
1963   if (is_inside && !sm->out2in_dpo)
1964     {
1965       if (sm->endpoint_dependent)
1966         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1967                                      sw_if_index, 1, 0, 0);
1968       else if (!sm->deterministic)
1969         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1970                                      sw_if_index, 1, 0, 0);
1971     }
1972
1973 set_flags:
1974   if (is_inside)
1975     {
1976       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1977       return 0;
1978     }
1979   else
1980     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1981
1982   /* Add/delete external addresses to FIB */
1983 fib:
1984   /* *INDENT-OFF* */
1985   vec_foreach (ap, sm->addresses)
1986     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1987
1988   pool_foreach (m, sm->static_mappings,
1989   ({
1990     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1991       continue;
1992
1993     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1994   }));
1995
1996   pool_foreach (dm, sm->det_maps,
1997   ({
1998     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1999   }));
2000   /* *INDENT-ON* */
2001
2002   return 0;
2003 }
2004
2005 int
2006 snat_interface_add_del_output_feature (u32 sw_if_index,
2007                                        u8 is_inside, int is_del)
2008 {
2009   snat_main_t *sm = &snat_main;
2010   snat_interface_t *i;
2011   snat_address_t *ap;
2012   snat_static_mapping_t *m;
2013   nat_outside_fib_t *outside_fib;
2014   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2015                                                        sw_if_index);
2016
2017
2018   if (sm->deterministic ||
2019       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2020     return VNET_API_ERROR_UNSUPPORTED;
2021
2022   /* *INDENT-OFF* */
2023   pool_foreach (i, sm->interfaces,
2024   ({
2025     if (i->sw_if_index == sw_if_index)
2026       return VNET_API_ERROR_VALUE_EXIST;
2027   }));
2028   /* *INDENT-ON* */
2029
2030   if (!is_inside)
2031     {
2032       /* *INDENT-OFF* */
2033       vec_foreach (outside_fib, sm->outside_fibs)
2034         {
2035           if (outside_fib->fib_index == fib_index)
2036             {
2037               if (is_del)
2038                 {
2039                   outside_fib->refcount--;
2040                   if (!outside_fib->refcount)
2041                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2042                 }
2043               else
2044                 outside_fib->refcount++;
2045               goto feature_set;
2046             }
2047         }
2048       /* *INDENT-ON* */
2049       if (!is_del)
2050         {
2051           vec_add2 (sm->outside_fibs, outside_fib, 1);
2052           outside_fib->refcount = 1;
2053           outside_fib->fib_index = fib_index;
2054         }
2055     }
2056
2057 feature_set:
2058   if (is_inside)
2059     {
2060       if (sm->endpoint_dependent)
2061         {
2062           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2063                                        sw_if_index, !is_del, 0, 0);
2064           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2065                                        sw_if_index, !is_del, 0, 0);
2066         }
2067       else
2068         {
2069           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2070                                        sw_if_index, !is_del, 0, 0);
2071           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2072                                        sw_if_index, !is_del, 0, 0);
2073         }
2074       goto fq;
2075     }
2076
2077   if (sm->num_workers > 1)
2078     {
2079       vnet_feature_enable_disable ("ip4-unicast",
2080                                    "nat44-out2in-worker-handoff",
2081                                    sw_if_index, !is_del, 0, 0);
2082       vnet_feature_enable_disable ("ip4-output",
2083                                    "nat44-in2out-output-worker-handoff",
2084                                    sw_if_index, !is_del, 0, 0);
2085     }
2086   else
2087     {
2088       if (sm->endpoint_dependent)
2089         {
2090           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2091                                        sw_if_index, !is_del, 0, 0);
2092           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2093                                        sw_if_index, !is_del, 0, 0);
2094         }
2095       else
2096         {
2097           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2098                                        sw_if_index, !is_del, 0, 0);
2099           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2100                                        sw_if_index, !is_del, 0, 0);
2101         }
2102     }
2103
2104 fq:
2105   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2106     sm->fq_in2out_output_index =
2107       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2108
2109   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2110     sm->fq_out2in_index =
2111       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2112
2113   /* *INDENT-OFF* */
2114   pool_foreach (i, sm->output_feature_interfaces,
2115   ({
2116     if (i->sw_if_index == sw_if_index)
2117       {
2118         if (is_del)
2119           pool_put (sm->output_feature_interfaces, i);
2120         else
2121           return VNET_API_ERROR_VALUE_EXIST;
2122
2123         goto fib;
2124       }
2125   }));
2126   /* *INDENT-ON* */
2127
2128   if (is_del)
2129     return VNET_API_ERROR_NO_SUCH_ENTRY;
2130
2131   pool_get (sm->output_feature_interfaces, i);
2132   i->sw_if_index = sw_if_index;
2133   i->flags = 0;
2134   if (is_inside)
2135     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2136   else
2137     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2138
2139   /* Add/delete external addresses to FIB */
2140 fib:
2141   if (is_inside)
2142     return 0;
2143
2144   /* *INDENT-OFF* */
2145   vec_foreach (ap, sm->addresses)
2146     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2147
2148   pool_foreach (m, sm->static_mappings,
2149   ({
2150     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2151       continue;
2152
2153     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2154   }));
2155   /* *INDENT-ON* */
2156
2157   return 0;
2158 }
2159
2160 int
2161 snat_set_workers (uword * bitmap)
2162 {
2163   snat_main_t *sm = &snat_main;
2164   int i, j = 0;
2165
2166   if (sm->num_workers < 2)
2167     return VNET_API_ERROR_FEATURE_DISABLED;
2168
2169   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2170     return VNET_API_ERROR_INVALID_WORKER;
2171
2172   vec_free (sm->workers);
2173   /* *INDENT-OFF* */
2174   clib_bitmap_foreach (i, bitmap,
2175     ({
2176       vec_add1(sm->workers, i);
2177       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2178       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2179       j++;
2180     }));
2181   /* *INDENT-ON* */
2182
2183   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2184   sm->num_snat_thread = _vec_len (sm->workers);
2185
2186   return 0;
2187 }
2188
2189 static void
2190 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2191                          u32 old_fib_index)
2192 {
2193   snat_main_t *sm = &snat_main;
2194   nat_outside_fib_t *outside_fib;
2195   snat_interface_t *i;
2196   u8 is_add = 1;
2197   u8 match = 0;
2198
2199   if (new_fib_index == old_fib_index)
2200     return;
2201
2202   if (!vec_len (sm->outside_fibs))
2203     return;
2204
2205   /* *INDENT-OFF* */
2206   pool_foreach (i, sm->interfaces,
2207     ({
2208       if (i->sw_if_index == sw_if_index)
2209         {
2210           if (!(nat_interface_is_outside (i)))
2211             return;
2212           match = 1;
2213         }
2214     }));
2215
2216   pool_foreach (i, sm->output_feature_interfaces,
2217     ({
2218       if (i->sw_if_index == sw_if_index)
2219         {
2220           if (!(nat_interface_is_outside (i)))
2221             return;
2222           match = 1;
2223         }
2224     }));
2225   /* *INDENT-ON* */
2226
2227   if (!match)
2228     return;
2229
2230   vec_foreach (outside_fib, sm->outside_fibs)
2231   {
2232     if (outside_fib->fib_index == old_fib_index)
2233       {
2234         outside_fib->refcount--;
2235         if (!outside_fib->refcount)
2236           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2237         break;
2238       }
2239   }
2240
2241   vec_foreach (outside_fib, sm->outside_fibs)
2242   {
2243     if (outside_fib->fib_index == new_fib_index)
2244       {
2245         outside_fib->refcount++;
2246         is_add = 0;
2247         break;
2248       }
2249   }
2250
2251   if (is_add)
2252     {
2253       vec_add2 (sm->outside_fibs, outside_fib, 1);
2254       outside_fib->refcount = 1;
2255       outside_fib->fib_index = new_fib_index;
2256     }
2257 }
2258
2259 static void
2260 snat_ip4_table_bind (ip4_main_t * im,
2261                      uword opaque,
2262                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2263 {
2264   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2265 }
2266
2267 static void
2268 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2269                                        uword opaque,
2270                                        u32 sw_if_index,
2271                                        ip4_address_t * address,
2272                                        u32 address_length,
2273                                        u32 if_address_index, u32 is_delete);
2274
2275 static void
2276 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2277                                  uword opaque,
2278                                  u32 sw_if_index,
2279                                  ip4_address_t * address,
2280                                  u32 address_length,
2281                                  u32 if_address_index, u32 is_delete);
2282
2283 static int
2284 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2285                                  u32 fib_index,
2286                                  u32 thread_index,
2287                                  snat_session_key_t * k,
2288                                  u16 port_per_thread, u32 snat_thread_index);
2289
2290 static clib_error_t *
2291 snat_init (vlib_main_t * vm)
2292 {
2293   snat_main_t *sm = &snat_main;
2294   clib_error_t *error = 0;
2295   ip4_main_t *im = &ip4_main;
2296   ip_lookup_main_t *lm = &im->lookup_main;
2297   uword *p;
2298   vlib_thread_registration_t *tr;
2299   vlib_thread_main_t *tm = vlib_get_thread_main ();
2300   uword *bitmap = 0;
2301   u32 i;
2302   ip4_add_del_interface_address_callback_t cb4;
2303   vlib_node_t *node;
2304
2305   sm->vlib_main = vm;
2306   sm->vnet_main = vnet_get_main ();
2307   sm->ip4_main = im;
2308   sm->ip4_lookup_main = lm;
2309   sm->api_main = &api_main;
2310   sm->first_worker_index = 0;
2311   sm->num_workers = 0;
2312   sm->num_snat_thread = 1;
2313   sm->workers = 0;
2314   sm->port_per_thread = 0xffff - 1024;
2315   sm->fq_in2out_index = ~0;
2316   sm->fq_in2out_output_index = ~0;
2317   sm->fq_out2in_index = ~0;
2318   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2319   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2320   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2321   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2322   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2323   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2324   sm->forwarding_enabled = 0;
2325   sm->log_class = vlib_log_register_class ("nat", 0);
2326   sm->log_level = SNAT_LOG_ERROR;
2327   sm->mss_clamping = 0;
2328
2329   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2330   sm->error_node_index = node->index;
2331
2332   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2333   sm->pre_in2out_node_index = node->index;
2334   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2335   sm->pre_out2in_node_index = node->index;
2336
2337   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2338   sm->pre_in2out_node_index = node->index;
2339
2340   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2341   sm->pre_out2in_node_index = node->index;
2342
2343   // TODO: output ?? (special node)
2344
2345   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2346   sm->in2out_node_index = node->index;
2347   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2348   sm->in2out_output_node_index = node->index;
2349   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2350   sm->in2out_fast_node_index = node->index;
2351   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2352   sm->in2out_slowpath_node_index = node->index;
2353   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2354   sm->in2out_slowpath_output_node_index = node->index;
2355   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-reass");
2356   sm->in2out_reass_node_index = node->index;
2357
2358   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2359   sm->ed_in2out_node_index = node->index;
2360   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2361   sm->ed_in2out_slowpath_node_index = node->index;
2362   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-reass");
2363   sm->ed_in2out_reass_node_index = node->index;
2364
2365   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2366   sm->out2in_node_index = node->index;
2367   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2368   sm->out2in_fast_node_index = node->index;
2369   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-reass");
2370   sm->out2in_reass_node_index = node->index;
2371
2372   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2373   sm->ed_out2in_node_index = node->index;
2374   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2375   sm->ed_out2in_slowpath_node_index = node->index;
2376   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-reass");
2377   sm->ed_out2in_reass_node_index = node->index;
2378
2379   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2380   sm->det_in2out_node_index = node->index;
2381   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2382   sm->det_out2in_node_index = node->index;
2383
2384   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2385   sm->hairpinning_node_index = node->index;
2386   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2387   sm->hairpin_dst_node_index = node->index;
2388   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2389   sm->hairpin_src_node_index = node->index;
2390   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2391   sm->ed_hairpinning_node_index = node->index;
2392   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2393   sm->ed_hairpin_dst_node_index = node->index;
2394   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2395   sm->ed_hairpin_src_node_index = node->index;
2396
2397   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2398   if (p)
2399     {
2400       tr = (vlib_thread_registration_t *) p[0];
2401       if (tr)
2402         {
2403           sm->num_workers = tr->count;
2404           sm->first_worker_index = tr->first_index;
2405         }
2406     }
2407
2408   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2409
2410   /* Use all available workers by default */
2411   if (sm->num_workers > 1)
2412     {
2413       for (i = 0; i < sm->num_workers; i++)
2414         bitmap = clib_bitmap_set (bitmap, i, 1);
2415       snat_set_workers (bitmap);
2416       clib_bitmap_free (bitmap);
2417     }
2418   else
2419     {
2420       sm->per_thread_data[0].snat_thread_index = 0;
2421     }
2422
2423   error = snat_api_init (vm, sm);
2424   if (error)
2425     return error;
2426
2427   /* Set up the interface address add/del callback */
2428   cb4.function = snat_ip4_add_del_interface_address_cb;
2429   cb4.function_opaque = 0;
2430
2431   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2432
2433   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2434   cb4.function_opaque = 0;
2435
2436   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2437
2438   nat_dpo_module_init ();
2439
2440   /* Init counters */
2441   sm->total_users.name = "total-users";
2442   sm->total_users.stat_segment_name = "/nat44/total-users";
2443   vlib_validate_simple_counter (&sm->total_users, 0);
2444   vlib_zero_simple_counter (&sm->total_users, 0);
2445   sm->total_sessions.name = "total-sessions";
2446   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2447   vlib_validate_simple_counter (&sm->total_sessions, 0);
2448   vlib_zero_simple_counter (&sm->total_sessions, 0);
2449
2450   /* Init IPFIX logging */
2451   snat_ipfix_logging_init (vm);
2452
2453   /* Init NAT64 */
2454   error = nat64_init (vm);
2455   if (error)
2456     return error;
2457
2458   dslite_init (vm);
2459
2460   nat66_init (vm);
2461
2462   ip4_table_bind_callback_t cbt4 = {
2463     .function = snat_ip4_table_bind,
2464   };
2465   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2466
2467   /* Init virtual fragmenentation reassembly */
2468   return nat_reass_init (vm);
2469 }
2470
2471 VLIB_INIT_FUNCTION (snat_init);
2472
2473 void
2474 snat_free_outside_address_and_port (snat_address_t * addresses,
2475                                     u32 thread_index, snat_session_key_t * k)
2476 {
2477   snat_address_t *a;
2478   u32 address_index;
2479   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2480
2481   for (address_index = 0; address_index < vec_len (addresses);
2482        address_index++)
2483     {
2484       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2485         break;
2486     }
2487
2488   ASSERT (address_index < vec_len (addresses));
2489
2490   a = addresses + address_index;
2491
2492   switch (k->protocol)
2493     {
2494 #define _(N, i, n, s) \
2495     case SNAT_PROTOCOL_##N: \
2496       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2497         port_host_byte_order) == 1); \
2498       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2499         port_host_byte_order, 0); \
2500       a->busy_##n##_ports--; \
2501       a->busy_##n##_ports_per_thread[thread_index]--; \
2502       break;
2503       foreach_snat_protocol
2504 #undef _
2505     default:
2506       nat_elog_info ("unknown protocol");
2507       return;
2508     }
2509 }
2510
2511 static int
2512 nat_set_outside_address_and_port (snat_address_t * addresses,
2513                                   u32 thread_index, snat_session_key_t * k)
2514 {
2515   snat_address_t *a = 0;
2516   u32 address_index;
2517   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2518
2519   for (address_index = 0; address_index < vec_len (addresses);
2520        address_index++)
2521     {
2522       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2523         continue;
2524
2525       a = addresses + address_index;
2526       switch (k->protocol)
2527         {
2528 #define _(N, j, n, s) \
2529         case SNAT_PROTOCOL_##N: \
2530           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2531             return VNET_API_ERROR_INSTANCE_IN_USE; \
2532           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2533           a->busy_##n##_ports_per_thread[thread_index]++; \
2534           a->busy_##n##_ports++; \
2535           return 0;
2536           foreach_snat_protocol
2537 #undef _
2538         default:
2539           nat_elog_info ("unknown protocol");
2540           return 1;
2541         }
2542     }
2543
2544   return VNET_API_ERROR_NO_SUCH_ENTRY;
2545 }
2546
2547 int
2548 snat_static_mapping_match (snat_main_t * sm,
2549                            snat_session_key_t match,
2550                            snat_session_key_t * mapping,
2551                            u8 by_external,
2552                            u8 * is_addr_only,
2553                            twice_nat_type_t * twice_nat,
2554                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2555                            u8 * is_identity_nat)
2556 {
2557   clib_bihash_kv_8_8_t kv, value;
2558   snat_static_mapping_t *m;
2559   snat_session_key_t m_key;
2560   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2561   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2562   u8 backend_index;
2563   nat44_lb_addr_port_t *local;
2564
2565   m_key.fib_index = match.fib_index;
2566   if (by_external)
2567     {
2568       mapping_hash = &sm->static_mapping_by_external;
2569       m_key.fib_index = 0;
2570     }
2571
2572   m_key.addr = match.addr;
2573   m_key.port = clib_net_to_host_u16 (match.port);
2574   m_key.protocol = match.protocol;
2575
2576   kv.key = m_key.as_u64;
2577
2578   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2579     {
2580       /* Try address only mapping */
2581       m_key.port = 0;
2582       m_key.protocol = 0;
2583       kv.key = m_key.as_u64;
2584       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2585         return 1;
2586     }
2587
2588   m = pool_elt_at_index (sm->static_mappings, value.value);
2589
2590   if (by_external)
2591     {
2592       if (is_lb_static_mapping (m))
2593         {
2594           if (PREDICT_FALSE (lb != 0))
2595             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2596           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2597                                                           match.addr,
2598                                                           match.protocol,
2599                                                           match.port,
2600                                                           &backend_index))
2601             {
2602               local = pool_elt_at_index (m->locals, backend_index);
2603               mapping->addr = local->addr;
2604               mapping->port = clib_host_to_net_u16 (local->port);
2605               mapping->fib_index = local->fib_index;
2606               goto end;
2607             }
2608           // pick locals matching this worker
2609           if (PREDICT_FALSE (sm->num_workers > 1))
2610             {
2611               u32 thread_index = vlib_get_thread_index ();
2612               /* *INDENT-OFF* */
2613               pool_foreach_index (i, m->locals,
2614               ({
2615                 local = pool_elt_at_index (m->locals, i);
2616
2617                 ip4_header_t ip = {
2618                   .src_address = local->addr,
2619                 };
2620
2621                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2622                     thread_index)
2623                   {
2624                     vec_add1 (tmp, i);
2625                   }
2626               }));
2627               /* *INDENT-ON* */
2628               ASSERT (vec_len (tmp) != 0);
2629             }
2630           else
2631             {
2632               /* *INDENT-OFF* */
2633               pool_foreach_index (i, m->locals,
2634               ({
2635                 vec_add1 (tmp, i);
2636               }));
2637               /* *INDENT-ON* */
2638             }
2639           hi = vec_len (tmp) - 1;
2640           local = pool_elt_at_index (m->locals, tmp[hi]);
2641           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2642           while (lo < hi)
2643             {
2644               mid = ((hi - lo) >> 1) + lo;
2645               local = pool_elt_at_index (m->locals, tmp[mid]);
2646               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2647             }
2648           local = pool_elt_at_index (m->locals, tmp[lo]);
2649           if (!(local->prefix >= rand))
2650             return 1;
2651           mapping->addr = local->addr;
2652           mapping->port = clib_host_to_net_u16 (local->port);
2653           mapping->fib_index = local->fib_index;
2654           if (m->affinity)
2655             {
2656               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2657                                                 match.protocol, match.port,
2658                                                 tmp[lo], m->affinity,
2659                                                 m->affinity_per_service_list_head_index))
2660                 nat_elog_info ("create affinity record failed");
2661             }
2662           vec_free (tmp);
2663         }
2664       else
2665         {
2666           if (PREDICT_FALSE (lb != 0))
2667             *lb = NO_LB_NAT;
2668           mapping->fib_index = m->fib_index;
2669           mapping->addr = m->local_addr;
2670           /* Address only mapping doesn't change port */
2671           mapping->port = is_addr_only_static_mapping (m) ? match.port
2672             : clib_host_to_net_u16 (m->local_port);
2673         }
2674       mapping->protocol = m->proto;
2675     }
2676   else
2677     {
2678       mapping->addr = m->external_addr;
2679       /* Address only mapping doesn't change port */
2680       mapping->port = is_addr_only_static_mapping (m) ? match.port
2681         : clib_host_to_net_u16 (m->external_port);
2682       mapping->fib_index = sm->outside_fib_index;
2683     }
2684
2685 end:
2686   if (PREDICT_FALSE (is_addr_only != 0))
2687     *is_addr_only = is_addr_only_static_mapping (m);
2688
2689   if (PREDICT_FALSE (twice_nat != 0))
2690     *twice_nat = m->twice_nat;
2691
2692   if (PREDICT_FALSE (is_identity_nat != 0))
2693     *is_identity_nat = is_identity_static_mapping (m);
2694
2695   return 0;
2696 }
2697
2698 static_always_inline u16
2699 snat_random_port (u16 min, u16 max)
2700 {
2701   snat_main_t *sm = &snat_main;
2702   return min + random_u32 (&sm->random_seed) /
2703     (random_u32_max () / (max - min + 1) + 1);
2704 }
2705
2706 int
2707 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2708                                      u32 fib_index,
2709                                      u32 thread_index,
2710                                      snat_session_key_t * k,
2711                                      u16 port_per_thread,
2712                                      u32 snat_thread_index)
2713 {
2714   snat_main_t *sm = &snat_main;
2715
2716   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2717                                   port_per_thread, snat_thread_index);
2718 }
2719
2720 static int
2721 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2722                                  u32 fib_index,
2723                                  u32 thread_index,
2724                                  snat_session_key_t * k,
2725                                  u16 port_per_thread, u32 snat_thread_index)
2726 {
2727   int i;
2728   snat_address_t *a, *ga = 0;
2729   u32 portnum;
2730
2731   for (i = 0; i < vec_len (addresses); i++)
2732     {
2733       a = addresses + i;
2734       switch (k->protocol)
2735         {
2736 #define _(N, j, n, s) \
2737         case SNAT_PROTOCOL_##N: \
2738           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2739             { \
2740               if (a->fib_index == fib_index) \
2741                 { \
2742                   while (1) \
2743                     { \
2744                       portnum = (port_per_thread * \
2745                         snat_thread_index) + \
2746                         snat_random_port(1, port_per_thread) + 1024; \
2747                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2748                         continue; \
2749                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2750                       a->busy_##n##_ports_per_thread[thread_index]++; \
2751                       a->busy_##n##_ports++; \
2752                       k->addr = a->addr; \
2753                       k->port = clib_host_to_net_u16(portnum); \
2754                       return 0; \
2755                     } \
2756                 } \
2757               else if (a->fib_index == ~0) \
2758                 { \
2759                   ga = a; \
2760                 } \
2761             } \
2762           break;
2763           foreach_snat_protocol
2764 #undef _
2765         default:
2766           nat_elog_info ("unknown protocol");
2767           return 1;
2768         }
2769
2770     }
2771
2772   if (ga)
2773     {
2774       a = ga;
2775       switch (k->protocol)
2776         {
2777 #define _(N, j, n, s) \
2778         case SNAT_PROTOCOL_##N: \
2779           while (1) \
2780             { \
2781               portnum = (port_per_thread * \
2782                 snat_thread_index) + \
2783                 snat_random_port(1, port_per_thread) + 1024; \
2784               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2785                 continue; \
2786               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2787               a->busy_##n##_ports_per_thread[thread_index]++; \
2788               a->busy_##n##_ports++; \
2789               k->addr = a->addr; \
2790               k->port = clib_host_to_net_u16(portnum); \
2791               return 0; \
2792             }
2793           break;
2794           foreach_snat_protocol
2795 #undef _
2796         default:
2797           nat_elog_info ("unknown protocol");
2798           return 1;
2799         }
2800     }
2801
2802   /* Totally out of translations to use... */
2803   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2804   return 1;
2805 }
2806
2807 static int
2808 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2809                               u32 fib_index,
2810                               u32 thread_index,
2811                               snat_session_key_t * k,
2812                               u16 port_per_thread, u32 snat_thread_index)
2813 {
2814   snat_main_t *sm = &snat_main;
2815   snat_address_t *a = addresses;
2816   u16 m, ports, portnum, A, j;
2817   m = 16 - (sm->psid_offset + sm->psid_length);
2818   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2819
2820   if (!vec_len (addresses))
2821     goto exhausted;
2822
2823   switch (k->protocol)
2824     {
2825 #define _(N, i, n, s) \
2826     case SNAT_PROTOCOL_##N: \
2827       if (a->busy_##n##_ports < ports) \
2828         { \
2829           while (1) \
2830             { \
2831               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2832               j = snat_random_port(0, pow2_mask(m)); \
2833               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2834               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2835                 continue; \
2836               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2837               a->busy_##n##_ports++; \
2838               k->addr = a->addr; \
2839               k->port = clib_host_to_net_u16 (portnum); \
2840               return 0; \
2841             } \
2842         } \
2843       break;
2844       foreach_snat_protocol
2845 #undef _
2846     default:
2847       nat_elog_info ("unknown protocol");
2848       return 1;
2849     }
2850
2851 exhausted:
2852   /* Totally out of translations to use... */
2853   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2854   return 1;
2855 }
2856
2857 static int
2858 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2859                                u32 fib_index,
2860                                u32 thread_index,
2861                                snat_session_key_t * k,
2862                                u16 port_per_thread, u32 snat_thread_index)
2863 {
2864   snat_main_t *sm = &snat_main;
2865   snat_address_t *a = addresses;
2866   u16 portnum, ports;
2867
2868   ports = sm->end_port - sm->start_port + 1;
2869
2870   if (!vec_len (addresses))
2871     goto exhausted;
2872
2873   switch (k->protocol)
2874     {
2875 #define _(N, i, n, s) \
2876     case SNAT_PROTOCOL_##N: \
2877       if (a->busy_##n##_ports < ports) \
2878         { \
2879           while (1) \
2880             { \
2881               portnum = snat_random_port(sm->start_port, sm->end_port); \
2882               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2883                 continue; \
2884               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2885               a->busy_##n##_ports++; \
2886               k->addr = a->addr; \
2887               k->port = clib_host_to_net_u16 (portnum); \
2888               return 0; \
2889             } \
2890         } \
2891       break;
2892       foreach_snat_protocol
2893 #undef _
2894     default:
2895       nat_elog_info ("unknown protocol");
2896       return 1;
2897     }
2898
2899 exhausted:
2900   /* Totally out of translations to use... */
2901   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2902   return 1;
2903 }
2904
2905 void
2906 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2907 {
2908   dpo_id_t dpo_v4 = DPO_INVALID;
2909   fib_prefix_t pfx = {
2910     .fp_proto = FIB_PROTOCOL_IP4,
2911     .fp_len = 32,
2912     .fp_addr.ip4.as_u32 = addr.as_u32,
2913   };
2914
2915   if (is_add)
2916     {
2917       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2918       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2919                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2920       dpo_reset (&dpo_v4);
2921     }
2922   else
2923     {
2924       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2925     }
2926 }
2927
2928 u8 *
2929 format_session_kvp (u8 * s, va_list * args)
2930 {
2931   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2932   snat_session_key_t k;
2933
2934   k.as_u64 = v->key;
2935
2936   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2937
2938   return s;
2939 }
2940
2941 u8 *
2942 format_static_mapping_kvp (u8 * s, va_list * args)
2943 {
2944   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2945   snat_session_key_t k;
2946
2947   k.as_u64 = v->key;
2948
2949   s = format (s, "%U static-mapping-index %llu",
2950               format_static_mapping_key, &k, v->value);
2951
2952   return s;
2953 }
2954
2955 u8 *
2956 format_user_kvp (u8 * s, va_list * args)
2957 {
2958   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2959   snat_user_key_t k;
2960
2961   k.as_u64 = v->key;
2962
2963   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2964               k.fib_index, v->value);
2965
2966   return s;
2967 }
2968
2969 u8 *
2970 format_ed_session_kvp (u8 * s, va_list * args)
2971 {
2972   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2973   nat_ed_ses_key_t k;
2974
2975   k.as_u64[0] = v->key[0];
2976   k.as_u64[1] = v->key[1];
2977
2978   s =
2979     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2980             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2981             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2982             format_ip_protocol, k.proto, k.fib_index, v->value);
2983
2984   return s;
2985 }
2986
2987 static u32
2988 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
2989                            u8 is_output)
2990 {
2991   snat_main_t *sm = &snat_main;
2992   u32 next_worker_index = 0;
2993   u32 hash;
2994
2995   next_worker_index = sm->first_worker_index;
2996   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2997     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
2998
2999   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3000     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3001   else
3002     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3003
3004   return next_worker_index;
3005 }
3006
3007 static u32
3008 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3009                            u8 is_output)
3010 {
3011   snat_main_t *sm = &snat_main;
3012   udp_header_t *udp;
3013   u16 port;
3014   snat_session_key_t m_key;
3015   clib_bihash_kv_8_8_t kv, value;
3016   snat_static_mapping_t *m;
3017   u32 proto;
3018   u32 next_worker_index = 0;
3019
3020   /* first try static mappings without port */
3021   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3022     {
3023       m_key.addr = ip0->dst_address;
3024       m_key.port = 0;
3025       m_key.protocol = 0;
3026       m_key.fib_index = rx_fib_index0;
3027       kv.key = m_key.as_u64;
3028       if (!clib_bihash_search_8_8
3029           (&sm->static_mapping_by_external, &kv, &value))
3030         {
3031           m = pool_elt_at_index (sm->static_mappings, value.value);
3032           return m->workers[0];
3033         }
3034     }
3035
3036   proto = ip_proto_to_snat_proto (ip0->protocol);
3037   udp = ip4_next_header (ip0);
3038   port = udp->dst_port;
3039
3040   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
3041     {
3042       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
3043         return vlib_get_thread_index ();
3044
3045       nat_reass_ip4_t *reass;
3046       reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
3047                                   ip0->fragment_id, ip0->protocol);
3048
3049       if (reass && (reass->thread_index != (u32) ~ 0))
3050         return reass->thread_index;
3051
3052       if (ip4_is_first_fragment (ip0))
3053         {
3054           reass =
3055             nat_ip4_reass_create (ip0->src_address, ip0->dst_address,
3056                                   ip0->fragment_id, ip0->protocol);
3057           if (!reass)
3058             goto no_reass;
3059
3060           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3061             {
3062               m_key.addr = ip0->dst_address;
3063               m_key.port = clib_net_to_host_u16 (port);
3064               m_key.protocol = proto;
3065               m_key.fib_index = rx_fib_index0;
3066               kv.key = m_key.as_u64;
3067               if (!clib_bihash_search_8_8
3068                   (&sm->static_mapping_by_external, &kv, &value))
3069                 {
3070                   m = pool_elt_at_index (sm->static_mappings, value.value);
3071                   reass->thread_index = m->workers[0];
3072                   return reass->thread_index;
3073                 }
3074             }
3075           reass->thread_index = sm->first_worker_index;
3076           reass->thread_index +=
3077             sm->workers[(clib_net_to_host_u16 (port) - 1024) /
3078                         sm->port_per_thread];
3079           return reass->thread_index;
3080         }
3081       else
3082         return vlib_get_thread_index ();
3083     }
3084
3085 no_reass:
3086   /* unknown protocol */
3087   if (PREDICT_FALSE (proto == ~0))
3088     {
3089       /* use current thread */
3090       return vlib_get_thread_index ();
3091     }
3092
3093   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3094     {
3095       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3096       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3097       if (!icmp_is_error_message (icmp))
3098         port = echo->identifier;
3099       else
3100         {
3101           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3102           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3103           void *l4_header = ip4_next_header (inner_ip);
3104           switch (proto)
3105             {
3106             case SNAT_PROTOCOL_ICMP:
3107               icmp = (icmp46_header_t *) l4_header;
3108               echo = (icmp_echo_header_t *) (icmp + 1);
3109               port = echo->identifier;
3110               break;
3111             case SNAT_PROTOCOL_UDP:
3112             case SNAT_PROTOCOL_TCP:
3113               port = ((tcp_udp_header_t *) l4_header)->src_port;
3114               break;
3115             default:
3116               return vlib_get_thread_index ();
3117             }
3118         }
3119     }
3120
3121   /* try static mappings with port */
3122   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3123     {
3124       m_key.addr = ip0->dst_address;
3125       m_key.port = clib_net_to_host_u16 (port);
3126       m_key.protocol = proto;
3127       m_key.fib_index = rx_fib_index0;
3128       kv.key = m_key.as_u64;
3129       if (!clib_bihash_search_8_8
3130           (&sm->static_mapping_by_external, &kv, &value))
3131         {
3132           m = pool_elt_at_index (sm->static_mappings, value.value);
3133           return m->workers[0];
3134         }
3135     }
3136
3137   /* worker by outside port */
3138   next_worker_index = sm->first_worker_index;
3139   next_worker_index +=
3140     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3141   return next_worker_index;
3142 }
3143
3144 static u32
3145 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3146                                u8 is_output)
3147 {
3148   snat_main_t *sm = &snat_main;
3149   u32 next_worker_index = sm->first_worker_index;
3150   u32 hash;
3151
3152   clib_bihash_kv_16_8_t kv16, value16;
3153   snat_main_per_thread_data_t *tsm;
3154   udp_header_t *udp;
3155
3156   if (PREDICT_FALSE (is_output))
3157     {
3158       u32 fib_index = sm->outside_fib_index;
3159       nat_outside_fib_t *outside_fib;
3160       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3161       fib_prefix_t pfx = {
3162         .fp_proto = FIB_PROTOCOL_IP4,
3163         .fp_len = 32,
3164         .fp_addr = {
3165                     .ip4.as_u32 = ip->dst_address.as_u32,
3166                     }
3167         ,
3168       };
3169
3170       udp = ip4_next_header (ip);
3171
3172       switch (vec_len (sm->outside_fibs))
3173         {
3174         case 0:
3175           fib_index = sm->outside_fib_index;
3176           break;
3177         case 1:
3178           fib_index = sm->outside_fibs[0].fib_index;
3179           break;
3180         default:
3181             /* *INDENT-OFF* */
3182             vec_foreach (outside_fib, sm->outside_fibs)
3183               {
3184                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3185                 if (FIB_NODE_INDEX_INVALID != fei)
3186                   {
3187                     if (fib_entry_get_resolving_interface (fei) != ~0)
3188                       {
3189                         fib_index = outside_fib->fib_index;
3190                         break;
3191                       }
3192                   }
3193               }
3194             /* *INDENT-ON* */
3195           break;
3196         }
3197
3198       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3199                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3200
3201       /* *INDENT-OFF* */
3202       vec_foreach (tsm, sm->per_thread_data)
3203         {
3204           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3205                                                       &kv16, &value16)))
3206             {
3207               next_worker_index += tsm->thread_index;
3208
3209               nat_elog_debug_handoff (
3210                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3211                 next_worker_index, fib_index,
3212                 clib_net_to_host_u32 (ip->src_address.as_u32),
3213                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3214
3215               return next_worker_index;
3216             }
3217         }
3218       /* *INDENT-ON* */
3219     }
3220
3221   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3222     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3223
3224   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3225     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3226   else
3227     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3228
3229   if (PREDICT_TRUE (!is_output))
3230     {
3231       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3232                               next_worker_index, rx_fib_index,
3233                               clib_net_to_host_u32 (ip->src_address.as_u32),
3234                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3235     }
3236   else
3237     {
3238       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3239                               next_worker_index, rx_fib_index,
3240                               clib_net_to_host_u32 (ip->src_address.as_u32),
3241                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3242     }
3243
3244   return next_worker_index;
3245 }
3246
3247 static u32
3248 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index,
3249                                u8 is_output)
3250 {
3251   snat_main_t *sm = &snat_main;
3252   clib_bihash_kv_8_8_t kv, value;
3253   clib_bihash_kv_16_8_t kv16, value16;
3254   snat_main_per_thread_data_t *tsm;
3255
3256   u32 proto, next_worker_index = 0;
3257   udp_header_t *udp;
3258   u16 port;
3259   snat_static_mapping_t *m;
3260   u32 hash;
3261
3262   proto = ip_proto_to_snat_proto (ip->protocol);
3263
3264   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3265     {
3266       udp = ip4_next_header (ip);
3267
3268       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3269                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3270
3271       /* *INDENT-OFF* */
3272       vec_foreach (tsm, sm->per_thread_data)
3273         {
3274           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3275                                                       &kv16, &value16)))
3276             {
3277               next_worker_index = sm->first_worker_index + tsm->thread_index;
3278               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3279                           next_worker_index, rx_fib_index,
3280                           clib_net_to_host_u32 (ip->src_address.as_u32),
3281                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3282               return next_worker_index;
3283             }
3284           }
3285         /* *INDENT-ON* */
3286     }
3287   else if (proto == SNAT_PROTOCOL_ICMP)
3288     {
3289       nat_ed_ses_key_t key;
3290
3291       if (!get_icmp_o2i_ed_key (ip, &key))
3292         {
3293
3294           key.fib_index = rx_fib_index;
3295           kv16.key[0] = key.as_u64[0];
3296           kv16.key[1] = key.as_u64[1];
3297
3298           /* *INDENT-OFF* */
3299           vec_foreach (tsm, sm->per_thread_data)
3300             {
3301               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3302                                                           &kv16, &value16)))
3303                 {
3304                   next_worker_index = sm->first_worker_index +
3305                                       tsm->thread_index;
3306                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3307                               next_worker_index, rx_fib_index,
3308                               clib_net_to_host_u32 (ip->src_address.as_u32),
3309                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3310                   return next_worker_index;
3311                 }
3312             }
3313           /* *INDENT-ON* */
3314         }
3315     }
3316
3317   /* first try static mappings without port */
3318   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3319     {
3320       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3321       if (!clib_bihash_search_8_8
3322           (&sm->static_mapping_by_external, &kv, &value))
3323         {
3324           m = pool_elt_at_index (sm->static_mappings, value.value);
3325           next_worker_index = m->workers[0];
3326           goto done;
3327         }
3328     }
3329
3330   /* unknown protocol */
3331   if (PREDICT_FALSE (proto == ~0))
3332     {
3333       /* use current thread */
3334       next_worker_index = vlib_get_thread_index ();
3335       goto done;
3336     }
3337
3338   udp = ip4_next_header (ip);
3339   port = udp->dst_port;
3340
3341   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3342     {
3343       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3344       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3345       if (!icmp_is_error_message (icmp))
3346         port = echo->identifier;
3347       else
3348         {
3349           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3350           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3351           void *l4_header = ip4_next_header (inner_ip);
3352           switch (proto)
3353             {
3354             case SNAT_PROTOCOL_ICMP:
3355               icmp = (icmp46_header_t *) l4_header;
3356               echo = (icmp_echo_header_t *) (icmp + 1);
3357               port = echo->identifier;
3358               break;
3359             case SNAT_PROTOCOL_UDP:
3360             case SNAT_PROTOCOL_TCP:
3361               port = ((tcp_udp_header_t *) l4_header)->src_port;
3362               break;
3363             default:
3364               next_worker_index = vlib_get_thread_index ();
3365               goto done;
3366             }
3367         }
3368     }
3369
3370   /* try static mappings with port */
3371   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3372     {
3373       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3374                   clib_net_to_host_u16 (port));
3375       if (!clib_bihash_search_8_8
3376           (&sm->static_mapping_by_external, &kv, &value))
3377         {
3378           m = pool_elt_at_index (sm->static_mappings, value.value);
3379           if (!is_lb_static_mapping (m))
3380             {
3381               next_worker_index = m->workers[0];
3382               goto done;
3383             }
3384
3385           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3386             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3387
3388           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3389             next_worker_index =
3390               m->workers[hash & (_vec_len (m->workers) - 1)];
3391           else
3392             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3393           goto done;
3394         }
3395     }
3396
3397   /* worker by outside port */
3398   next_worker_index = sm->first_worker_index;
3399   next_worker_index +=
3400     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3401
3402 done:
3403   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3404                           clib_net_to_host_u32 (ip->src_address.as_u32),
3405                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3406   return next_worker_index;
3407 }
3408
3409 void
3410 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3411                 ip4_address_t * out_addr, u16 out_port,
3412                 ip4_address_t * eh_addr, u16 eh_port,
3413                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3414                 u32 fib_index, u16 flags, u32 thread_index)
3415 {
3416   snat_main_t *sm = &snat_main;
3417   snat_session_key_t key;
3418   snat_user_t *u;
3419   snat_session_t *s;
3420   clib_bihash_kv_8_8_t kv;
3421   f64 now = vlib_time_now (sm->vlib_main);
3422   nat_outside_fib_t *outside_fib;
3423   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3424   snat_main_per_thread_data_t *tsm;
3425   fib_prefix_t pfx = {
3426     .fp_proto = FIB_PROTOCOL_IP4,
3427     .fp_len = 32,
3428     .fp_addr = {
3429                 .ip4.as_u32 = eh_addr->as_u32,
3430                 },
3431   };
3432
3433   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3434
3435   key.addr.as_u32 = out_addr->as_u32;
3436   key.port = out_port;
3437   key.protocol = proto;
3438
3439   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3440     {
3441       if (nat_set_outside_address_and_port
3442           (sm->addresses, thread_index, &key))
3443         return;
3444     }
3445
3446   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3447   if (!u)
3448     return;
3449
3450   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3451   if (!s)
3452     return;
3453
3454   s->last_heard = now;
3455   s->flags = flags;
3456   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3457   s->ext_host_port = eh_port;
3458   user_session_increment (sm, u, snat_is_session_static (s));
3459   switch (vec_len (sm->outside_fibs))
3460     {
3461     case 0:
3462       key.fib_index = sm->outside_fib_index;
3463       break;
3464     case 1:
3465       key.fib_index = sm->outside_fibs[0].fib_index;
3466       break;
3467     default:
3468       /* *INDENT-OFF* */
3469       vec_foreach (outside_fib, sm->outside_fibs)
3470         {
3471           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3472           if (FIB_NODE_INDEX_INVALID != fei)
3473             {
3474               if (fib_entry_get_resolving_interface (fei) != ~0)
3475                 {
3476                   key.fib_index = outside_fib->fib_index;
3477                   break;
3478                 }
3479             }
3480         }
3481       /* *INDENT-ON* */
3482       break;
3483     }
3484   s->out2in = key;
3485   kv.key = key.as_u64;
3486   kv.value = s - tsm->sessions;
3487   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3488     nat_elog_warn ("out2in key add failed");
3489
3490   key.addr.as_u32 = in_addr->as_u32;
3491   key.port = in_port;
3492   key.fib_index = fib_index;
3493   s->in2out = key;
3494   kv.key = key.as_u64;
3495   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3496     nat_elog_warn ("in2out key add failed");
3497 }
3498
3499 void
3500 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3501                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3502                 u32 ti)
3503 {
3504   snat_main_t *sm = &snat_main;
3505   snat_session_key_t key;
3506   clib_bihash_kv_8_8_t kv, value;
3507   u32 thread_index;
3508   snat_session_t *s;
3509   snat_main_per_thread_data_t *tsm;
3510
3511   if (sm->num_workers > 1)
3512     thread_index =
3513       sm->first_worker_index +
3514       (sm->workers[(clib_net_to_host_u16 (out_port) -
3515                     1024) / sm->port_per_thread]);
3516   else
3517     thread_index = sm->num_workers;
3518   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3519
3520   key.addr.as_u32 = out_addr->as_u32;
3521   key.port = out_port;
3522   key.protocol = proto;
3523   key.fib_index = fib_index;
3524   kv.key = key.as_u64;
3525   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3526     return;
3527
3528   s = pool_elt_at_index (tsm->sessions, value.value);
3529   nat_free_session_data (sm, s, thread_index, 1);
3530   nat44_delete_session (sm, s, thread_index);
3531 }
3532
3533 void
3534 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3535                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3536                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3537 {
3538   snat_main_t *sm = &snat_main;
3539   snat_session_key_t key;
3540   clib_bihash_kv_8_8_t kv, value;
3541   snat_session_t *s;
3542   snat_main_per_thread_data_t *tsm;
3543
3544   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3545
3546   key.addr.as_u32 = out_addr->as_u32;
3547   key.port = out_port;
3548   key.protocol = proto;
3549   key.fib_index = fib_index;
3550   kv.key = key.as_u64;
3551   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3552     return;
3553
3554   s = pool_elt_at_index (tsm->sessions, value.value);
3555   s->total_pkts = total_pkts;
3556   s->total_bytes = total_bytes;
3557 }
3558
3559 void
3560 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3561                    ip4_address_t * out_addr, u16 out_port,
3562                    ip4_address_t * eh_addr, u16 eh_port,
3563                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3564                    u32 fib_index, u16 flags, u32 thread_index)
3565 {
3566   snat_main_t *sm = &snat_main;
3567   snat_session_key_t key;
3568   snat_user_t *u;
3569   snat_session_t *s;
3570   clib_bihash_kv_16_8_t kv;
3571   f64 now = vlib_time_now (sm->vlib_main);
3572   nat_outside_fib_t *outside_fib;
3573   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3574   snat_main_per_thread_data_t *tsm;
3575   fib_prefix_t pfx = {
3576     .fp_proto = FIB_PROTOCOL_IP4,
3577     .fp_len = 32,
3578     .fp_addr = {
3579                 .ip4.as_u32 = eh_addr->as_u32,
3580                 },
3581   };
3582
3583   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3584
3585   key.addr.as_u32 = out_addr->as_u32;
3586   key.port = out_port;
3587   key.protocol = proto;
3588
3589   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3590     {
3591       if (nat_set_outside_address_and_port
3592           (sm->addresses, thread_index, &key))
3593         return;
3594     }
3595
3596   key.addr.as_u32 = ehn_addr->as_u32;
3597   key.port = ehn_port;
3598   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3599     {
3600       if (nat_set_outside_address_and_port
3601           (sm->twice_nat_addresses, thread_index, &key))
3602         return;
3603     }
3604
3605   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3606   if (!u)
3607     return;
3608
3609   s = nat_ed_session_alloc (sm, u, thread_index, now);
3610   if (!s)
3611     return;
3612
3613   s->last_heard = now;
3614   s->flags = flags;
3615   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3616   s->ext_host_nat_port = s->ext_host_port = eh_port;
3617   if (is_twice_nat_session (s))
3618     {
3619       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3620       s->ext_host_nat_port = ehn_port;
3621     }
3622   user_session_increment (sm, u, snat_is_session_static (s));
3623   switch (vec_len (sm->outside_fibs))
3624     {
3625     case 0:
3626       key.fib_index = sm->outside_fib_index;
3627       break;
3628     case 1:
3629       key.fib_index = sm->outside_fibs[0].fib_index;
3630       break;
3631     default:
3632       /* *INDENT-OFF* */
3633       vec_foreach (outside_fib, sm->outside_fibs)
3634         {
3635           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3636           if (FIB_NODE_INDEX_INVALID != fei)
3637             {
3638               if (fib_entry_get_resolving_interface (fei) != ~0)
3639                 {
3640                   key.fib_index = outside_fib->fib_index;
3641                   break;
3642                 }
3643             }
3644         }
3645       /* *INDENT-ON* */
3646       break;
3647     }
3648   key.addr.as_u32 = out_addr->as_u32;
3649   key.port = out_port;
3650   s->out2in = key;
3651   kv.value = s - tsm->sessions;
3652
3653   key.addr.as_u32 = in_addr->as_u32;
3654   key.port = in_port;
3655   key.fib_index = fib_index;
3656   s->in2out = key;
3657
3658   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3659               snat_proto_to_ip_proto (proto), fib_index, in_port,
3660               s->ext_host_nat_port);
3661   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3662     nat_elog_warn ("in2out key add failed");
3663
3664   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3665               s->out2in.fib_index, out_port, eh_port);
3666   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3667     nat_elog_warn ("out2in key add failed");
3668 }
3669
3670 void
3671 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3672                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3673                    u32 fib_index, u32 ti)
3674 {
3675   snat_main_t *sm = &snat_main;
3676   nat_ed_ses_key_t key;
3677   clib_bihash_kv_16_8_t kv, value;
3678   u32 thread_index;
3679   snat_session_t *s;
3680   snat_main_per_thread_data_t *tsm;
3681
3682   if (sm->num_workers > 1)
3683     thread_index =
3684       sm->first_worker_index +
3685       (sm->workers[(clib_net_to_host_u16 (out_port) -
3686                     1024) / sm->port_per_thread]);
3687   else
3688     thread_index = sm->num_workers;
3689   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3690
3691   key.l_addr.as_u32 = out_addr->as_u32;
3692   key.l_port = out_port;
3693   key.r_addr.as_u32 = eh_addr->as_u32;
3694   key.r_port = eh_port;
3695   key.proto = proto;
3696   key.fib_index = fib_index;
3697   kv.key[0] = key.as_u64[0];
3698   kv.key[1] = key.as_u64[1];
3699   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3700     return;
3701
3702   s = pool_elt_at_index (tsm->sessions, value.value);
3703   nat_free_session_data (sm, s, thread_index, 1);
3704   nat44_delete_session (sm, s, thread_index);
3705 }
3706
3707 void
3708 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3709                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3710                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3711                    u32 thread_index)
3712 {
3713   snat_main_t *sm = &snat_main;
3714   nat_ed_ses_key_t key;
3715   clib_bihash_kv_16_8_t kv, value;
3716   snat_session_t *s;
3717   snat_main_per_thread_data_t *tsm;
3718
3719   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3720
3721   key.l_addr.as_u32 = out_addr->as_u32;
3722   key.l_port = out_port;
3723   key.r_addr.as_u32 = eh_addr->as_u32;
3724   key.r_port = eh_port;
3725   key.proto = proto;
3726   key.fib_index = fib_index;
3727   kv.key[0] = key.as_u64[0];
3728   kv.key[1] = key.as_u64[1];
3729   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3730     return;
3731
3732   s = pool_elt_at_index (tsm->sessions, value.value);
3733   s->total_pkts = total_pkts;
3734   s->total_bytes = total_bytes;
3735 }
3736
3737 static clib_error_t *
3738 snat_config (vlib_main_t * vm, unformat_input_t * input)
3739 {
3740   snat_main_t *sm = &snat_main;
3741   nat66_main_t *nm = &nat66_main;
3742   u32 translation_buckets = 1024;
3743   u32 translation_memory_size = 128 << 20;
3744   u32 user_buckets = 128;
3745   u32 user_memory_size = 64 << 20;
3746   u32 max_translations_per_user = 100;
3747   u32 outside_vrf_id = 0;
3748   u32 outside_ip6_vrf_id = 0;
3749   u32 inside_vrf_id = 0;
3750   u32 static_mapping_buckets = 1024;
3751   u32 static_mapping_memory_size = 64 << 20;
3752   u32 nat64_bib_buckets = 1024;
3753   u32 nat64_bib_memory_size = 128 << 20;
3754   u32 nat64_st_buckets = 2048;
3755   u32 nat64_st_memory_size = 256 << 20;
3756   u8 static_mapping_only = 0;
3757   u8 static_mapping_connection_tracking = 0;
3758   snat_main_per_thread_data_t *tsm;
3759   dslite_main_t *dm = &dslite_main;
3760
3761   sm->deterministic = 0;
3762   sm->out2in_dpo = 0;
3763   sm->endpoint_dependent = 0;
3764
3765   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3766     {
3767       if (unformat
3768           (input, "translation hash buckets %d", &translation_buckets))
3769         ;
3770       else if (unformat (input, "translation hash memory %d",
3771                          &translation_memory_size));
3772       else if (unformat (input, "user hash buckets %d", &user_buckets))
3773         ;
3774       else if (unformat (input, "user hash memory %d", &user_memory_size))
3775         ;
3776       else if (unformat (input, "max translations per user %d",
3777                          &max_translations_per_user))
3778         ;
3779       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3780         ;
3781       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3782         ;
3783       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3784         ;
3785       else if (unformat (input, "static mapping only"))
3786         {
3787           static_mapping_only = 1;
3788           if (unformat (input, "connection tracking"))
3789             static_mapping_connection_tracking = 1;
3790         }
3791       else if (unformat (input, "deterministic"))
3792         sm->deterministic = 1;
3793       else if (unformat (input, "nat64 bib hash buckets %d",
3794                          &nat64_bib_buckets))
3795         ;
3796       else if (unformat (input, "nat64 bib hash memory %d",
3797                          &nat64_bib_memory_size))
3798         ;
3799       else
3800         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3801         ;
3802       else if (unformat (input, "nat64 st hash memory %d",
3803                          &nat64_st_memory_size))
3804         ;
3805       else if (unformat (input, "out2in dpo"))
3806         sm->out2in_dpo = 1;
3807       else if (unformat (input, "dslite ce"))
3808         dslite_set_ce (dm, 1);
3809       else if (unformat (input, "endpoint-dependent"))
3810         sm->endpoint_dependent = 1;
3811       else
3812         return clib_error_return (0, "unknown input '%U'",
3813                                   format_unformat_error, input);
3814     }
3815
3816   if (sm->deterministic && sm->endpoint_dependent)
3817     return clib_error_return (0,
3818                               "deterministic and endpoint-dependent modes are mutually exclusive");
3819
3820   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3821     return clib_error_return (0,
3822                               "static mapping only mode available only for simple nat");
3823
3824   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3825     return clib_error_return (0,
3826                               "out2in dpo mode available only for simple nat");
3827
3828   /* for show commands, etc. */
3829   sm->translation_buckets = translation_buckets;
3830   sm->translation_memory_size = translation_memory_size;
3831   /* do not exceed load factor 10 */
3832   sm->max_translations = 10 * translation_buckets;
3833   sm->user_buckets = user_buckets;
3834   sm->user_memory_size = user_memory_size;
3835   sm->max_translations_per_user = max_translations_per_user;
3836   sm->outside_vrf_id = outside_vrf_id;
3837   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3838                                                              outside_vrf_id,
3839                                                              FIB_SOURCE_PLUGIN_HI);
3840   nm->outside_vrf_id = outside_ip6_vrf_id;
3841   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3842                                                              outside_ip6_vrf_id,
3843                                                              FIB_SOURCE_PLUGIN_HI);
3844   sm->inside_vrf_id = inside_vrf_id;
3845   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3846                                                             inside_vrf_id,
3847                                                             FIB_SOURCE_PLUGIN_HI);
3848   sm->static_mapping_only = static_mapping_only;
3849   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3850
3851   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3852                   nat64_st_memory_size);
3853
3854   if (sm->deterministic)
3855     {
3856       sm->in2out_node_index = snat_det_in2out_node.index;
3857       sm->in2out_output_node_index = ~0;
3858       sm->out2in_node_index = snat_det_out2in_node.index;
3859       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3860       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3861     }
3862   else
3863     {
3864       if (sm->endpoint_dependent)
3865         {
3866           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
3867           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3868
3869           sm->handoff_out2in_index = nat_pre_out2in_node.index;
3870           sm->handoff_in2out_index = nat_pre_in2out_node.index;
3871           // TODO: test
3872           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
3873
3874           sm->in2out_node_index = nat44_ed_in2out_node.index;
3875           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3876           sm->out2in_node_index = nat44_ed_out2in_node.index;
3877
3878           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3879           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3880           nat_affinity_init (vm);
3881           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
3882                        nat_ha_sref_ed_cb);
3883         }
3884       else
3885         {
3886           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3887           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3888
3889           sm->handoff_out2in_index = snat_in2out_node.index;
3890           sm->handoff_in2out_index = snat_out2in_node.index;
3891           // TODO: test
3892           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
3893
3894           sm->in2out_node_index = snat_in2out_node.index;
3895           sm->in2out_output_node_index = snat_in2out_output_node.index;
3896           sm->out2in_node_index = snat_out2in_node.index;
3897           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3898           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3899           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
3900         }
3901       if (!static_mapping_only ||
3902           (static_mapping_only && static_mapping_connection_tracking))
3903         {
3904           /* *INDENT-OFF* */
3905           vec_foreach (tsm, sm->per_thread_data)
3906             {
3907               if (sm->endpoint_dependent)
3908                 {
3909                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3910                                          translation_buckets,
3911                                          translation_memory_size);
3912                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3913                                                       format_ed_session_kvp);
3914
3915                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3916                                          translation_buckets,
3917                                          translation_memory_size);
3918                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3919                                                       format_ed_session_kvp);
3920                 }
3921               else
3922                 {
3923                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3924                                         translation_buckets,
3925                                         translation_memory_size);
3926                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3927                                                      format_session_kvp);
3928
3929                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3930                                         translation_buckets,
3931                                         translation_memory_size);
3932                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3933                                                      format_session_kvp);
3934                 }
3935
3936               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3937                                     user_memory_size);
3938               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3939                                                  format_user_kvp);
3940             }
3941           /* *INDENT-ON* */
3942
3943         }
3944       else
3945         {
3946           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3947           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3948         }
3949       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3950                             "static_mapping_by_local", static_mapping_buckets,
3951                             static_mapping_memory_size);
3952       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3953                                          format_static_mapping_kvp);
3954
3955       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3956                             "static_mapping_by_external",
3957                             static_mapping_buckets,
3958                             static_mapping_memory_size);
3959       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3960                                          format_static_mapping_kvp);
3961     }
3962
3963   return 0;
3964 }
3965
3966 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3967
3968 static void
3969 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3970                                  uword opaque,
3971                                  u32 sw_if_index,
3972                                  ip4_address_t * address,
3973                                  u32 address_length,
3974                                  u32 if_address_index, u32 is_delete)
3975 {
3976   snat_main_t *sm = &snat_main;
3977   snat_static_map_resolve_t *rp;
3978   snat_static_mapping_t *m;
3979   snat_session_key_t m_key;
3980   clib_bihash_kv_8_8_t kv, value;
3981   int i, rv;
3982   ip4_address_t l_addr;
3983
3984   for (i = 0; i < vec_len (sm->to_resolve); i++)
3985     {
3986       rp = sm->to_resolve + i;
3987       if (rp->addr_only == 0)
3988         continue;
3989       if (rp->sw_if_index == sw_if_index)
3990         goto match;
3991     }
3992
3993   return;
3994
3995 match:
3996   m_key.addr.as_u32 = address->as_u32;
3997   m_key.port = rp->addr_only ? 0 : rp->e_port;
3998   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3999   m_key.fib_index = sm->outside_fib_index;
4000   kv.key = m_key.as_u64;
4001   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4002     m = 0;
4003   else
4004     m = pool_elt_at_index (sm->static_mappings, value.value);
4005
4006   if (!is_delete)
4007     {
4008       /* Don't trip over lease renewal, static config */
4009       if (m)
4010         return;
4011     }
4012   else
4013     {
4014       if (!m)
4015         return;
4016     }
4017
4018   /* Indetity mapping? */
4019   if (rp->l_addr.as_u32 == 0)
4020     l_addr.as_u32 = address[0].as_u32;
4021   else
4022     l_addr.as_u32 = rp->l_addr.as_u32;
4023   /* Add the static mapping */
4024   rv = snat_add_static_mapping (l_addr,
4025                                 address[0],
4026                                 rp->l_port,
4027                                 rp->e_port,
4028                                 rp->vrf_id,
4029                                 rp->addr_only, ~0 /* sw_if_index */ ,
4030                                 rp->proto, !is_delete, rp->twice_nat,
4031                                 rp->out2in_only, rp->tag, rp->identity_nat);
4032   if (rv)
4033     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4034 }
4035
4036 static void
4037 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4038                                        uword opaque,
4039                                        u32 sw_if_index,
4040                                        ip4_address_t * address,
4041                                        u32 address_length,
4042                                        u32 if_address_index, u32 is_delete)
4043 {
4044   snat_main_t *sm = &snat_main;
4045   snat_static_map_resolve_t *rp;
4046   ip4_address_t l_addr;
4047   int i, j;
4048   int rv;
4049   u8 twice_nat = 0;
4050   snat_address_t *addresses = sm->addresses;
4051
4052   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4053     {
4054       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4055         goto match;
4056     }
4057
4058   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4059     {
4060       twice_nat = 1;
4061       addresses = sm->twice_nat_addresses;
4062       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4063         goto match;
4064     }
4065
4066   return;
4067
4068 match:
4069   if (!is_delete)
4070     {
4071       /* Don't trip over lease renewal, static config */
4072       for (j = 0; j < vec_len (addresses); j++)
4073         if (addresses[j].addr.as_u32 == address->as_u32)
4074           return;
4075
4076       (void) snat_add_address (sm, address, ~0, twice_nat);
4077       /* Scan static map resolution vector */
4078       for (j = 0; j < vec_len (sm->to_resolve); j++)
4079         {
4080           rp = sm->to_resolve + j;
4081           if (rp->addr_only)
4082             continue;
4083           /* On this interface? */
4084           if (rp->sw_if_index == sw_if_index)
4085             {
4086               /* Indetity mapping? */
4087               if (rp->l_addr.as_u32 == 0)
4088                 l_addr.as_u32 = address[0].as_u32;
4089               else
4090                 l_addr.as_u32 = rp->l_addr.as_u32;
4091               /* Add the static mapping */
4092               rv = snat_add_static_mapping (l_addr,
4093                                             address[0],
4094                                             rp->l_port,
4095                                             rp->e_port,
4096                                             rp->vrf_id,
4097                                             rp->addr_only,
4098                                             ~0 /* sw_if_index */ ,
4099                                             rp->proto,
4100                                             rp->is_add, rp->twice_nat,
4101                                             rp->out2in_only, rp->tag,
4102                                             rp->identity_nat);
4103               if (rv)
4104                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4105                                     "i4", rv);
4106             }
4107         }
4108       return;
4109     }
4110   else
4111     {
4112       (void) snat_del_address (sm, address[0], 1, twice_nat);
4113       return;
4114     }
4115 }
4116
4117
4118 int
4119 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4120                             u8 twice_nat)
4121 {
4122   ip4_main_t *ip4_main = sm->ip4_main;
4123   ip4_address_t *first_int_addr;
4124   snat_static_map_resolve_t *rp;
4125   u32 *indices_to_delete = 0;
4126   int i, j;
4127   u32 *auto_add_sw_if_indices =
4128     twice_nat ? sm->
4129     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4130
4131   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4132     );
4133
4134   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4135     {
4136       if (auto_add_sw_if_indices[i] == sw_if_index)
4137         {
4138           if (is_del)
4139             {
4140               /* if have address remove it */
4141               if (first_int_addr)
4142                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4143               else
4144                 {
4145                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4146                     {
4147                       rp = sm->to_resolve + j;
4148                       if (rp->sw_if_index == sw_if_index)
4149                         vec_add1 (indices_to_delete, j);
4150                     }
4151                   if (vec_len (indices_to_delete))
4152                     {
4153                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4154                         vec_del1 (sm->to_resolve, j);
4155                       vec_free (indices_to_delete);
4156                     }
4157                 }
4158               if (twice_nat)
4159                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4160               else
4161                 vec_del1 (sm->auto_add_sw_if_indices, i);
4162             }
4163           else
4164             return VNET_API_ERROR_VALUE_EXIST;
4165
4166           return 0;
4167         }
4168     }
4169
4170   if (is_del)
4171     return VNET_API_ERROR_NO_SUCH_ENTRY;
4172
4173   /* add to the auto-address list */
4174   if (twice_nat)
4175     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4176   else
4177     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4178
4179   /* If the address is already bound - or static - add it now */
4180   if (first_int_addr)
4181     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4182
4183   return 0;
4184 }
4185
4186 int
4187 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4188                    snat_protocol_t proto, u32 vrf_id, int is_in)
4189 {
4190   snat_main_per_thread_data_t *tsm;
4191   clib_bihash_kv_8_8_t kv, value;
4192   ip4_header_t ip;
4193   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4194   snat_session_key_t key;
4195   snat_session_t *s;
4196   clib_bihash_8_8_t *t;
4197
4198   if (sm->endpoint_dependent)
4199     return VNET_API_ERROR_UNSUPPORTED;
4200
4201   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4202   if (sm->num_workers > 1)
4203     tsm =
4204       vec_elt_at_index (sm->per_thread_data,
4205                         sm->worker_in2out_cb (&ip, fib_index, 0));
4206   else
4207     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4208
4209   key.addr.as_u32 = addr->as_u32;
4210   key.port = clib_host_to_net_u16 (port);
4211   key.protocol = proto;
4212   key.fib_index = fib_index;
4213   kv.key = key.as_u64;
4214   t = is_in ? &tsm->in2out : &tsm->out2in;
4215   if (!clib_bihash_search_8_8 (t, &kv, &value))
4216     {
4217       if (pool_is_free_index (tsm->sessions, value.value))
4218         return VNET_API_ERROR_UNSPECIFIED;
4219
4220       s = pool_elt_at_index (tsm->sessions, value.value);
4221       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4222       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4223       return 0;
4224     }
4225
4226   return VNET_API_ERROR_NO_SUCH_ENTRY;
4227 }
4228
4229 int
4230 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4231                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4232                       u32 vrf_id, int is_in)
4233 {
4234   ip4_header_t ip;
4235   clib_bihash_16_8_t *t;
4236   nat_ed_ses_key_t key;
4237   clib_bihash_kv_16_8_t kv, value;
4238   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4239   snat_session_t *s;
4240   snat_main_per_thread_data_t *tsm;
4241
4242   if (!sm->endpoint_dependent)
4243     return VNET_API_ERROR_FEATURE_DISABLED;
4244
4245   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4246   if (sm->num_workers > 1)
4247     tsm =
4248       vec_elt_at_index (sm->per_thread_data,
4249                         sm->worker_in2out_cb (&ip, fib_index, 0));
4250   else
4251     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4252
4253   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4254   key.l_addr.as_u32 = addr->as_u32;
4255   key.r_addr.as_u32 = eh_addr->as_u32;
4256   key.l_port = clib_host_to_net_u16 (port);
4257   key.r_port = clib_host_to_net_u16 (eh_port);
4258   key.proto = proto;
4259   key.fib_index = fib_index;
4260   kv.key[0] = key.as_u64[0];
4261   kv.key[1] = key.as_u64[1];
4262   if (clib_bihash_search_16_8 (t, &kv, &value))
4263     return VNET_API_ERROR_NO_SUCH_ENTRY;
4264
4265   if (pool_is_free_index (tsm->sessions, value.value))
4266     return VNET_API_ERROR_UNSPECIFIED;
4267   s = pool_elt_at_index (tsm->sessions, value.value);
4268   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4269   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4270   return 0;
4271 }
4272
4273 void
4274 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4275 {
4276   snat_main_t *sm = &snat_main;
4277
4278   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4279   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4280   sm->psid = psid;
4281   sm->psid_offset = psid_offset;
4282   sm->psid_length = psid_length;
4283 }
4284
4285 void
4286 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4287 {
4288   snat_main_t *sm = &snat_main;
4289
4290   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4291   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4292   sm->start_port = start_port;
4293   sm->end_port = end_port;
4294 }
4295
4296 void
4297 nat_set_alloc_addr_and_port_default (void)
4298 {
4299   snat_main_t *sm = &snat_main;
4300
4301   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4302   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4303 }
4304
4305 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4306                                  vlib_node_runtime_t * node,
4307                                  vlib_frame_t * frame)
4308 {
4309   return 0;
4310 }
4311
4312 /* *INDENT-OFF* */
4313 VLIB_REGISTER_NODE (nat_default_node) = {
4314   .name = "nat-default",
4315   .vector_size = sizeof (u32),
4316   .format_trace = 0,
4317   .type = VLIB_NODE_TYPE_INTERNAL,
4318   .n_errors = 0,
4319   .n_next_nodes = NAT_N_NEXT,
4320   .next_nodes = {
4321     [NAT_NEXT_DROP] = "error-drop",
4322     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4323     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4324     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4325     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4326     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4327     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4328     [NAT_NEXT_IN2OUT_ED_REASS] = "nat44-ed-in2out-reass",
4329     [NAT_NEXT_IN2OUT_ED_OUTPUT_REASS] = "nat44-ed-in2out-reass-output",
4330     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4331     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4332     [NAT_NEXT_OUT2IN_ED_REASS] = "nat44-ed-out2in-reass",
4333   },
4334 };
4335 /* *INDENT-ON* */
4336
4337 /*
4338  * fd.io coding-style-patch-verification: ON
4339  *
4340  * Local Variables:
4341  * eval: (c-set-style "gnu")
4342  * End:
4343  */