src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <vnet/fib/fib_table.h>
  35 #include <vnet/fib/ip4_fib.h>
  36
  37 #include <vpp/app/version.h>
  38
  39 snat_main_t snat_main;
  40
  41 fib_source_t nat_fib_src_hi;
  42 fib_source_t nat_fib_src_low;
  43
  44 /* *INDENT-OFF* */
  45 /* Hook up input features */
  46 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat-pre-in2out",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  50 };
  51 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  52   .arc_name = "ip4-unicast",
  53   .node_name = "nat-pre-out2in",
  54   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  55                                "ip4-dhcp-client-detect"),
  56 };
  57 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  58   .arc_name = "ip4-unicast",
  59   .node_name = "nat44-in2out-worker-handoff",
  60   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  61 };
  62 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  63   .arc_name = "ip4-unicast",
  64   .node_name = "nat44-out2in-worker-handoff",
  65   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  66                                "ip4-dhcp-client-detect"),
  67 };
  68 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  69   .arc_name = "ip4-unicast",
  70   .node_name = "nat44-in2out",
  71   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  72 };
  73 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  74   .arc_name = "ip4-unicast",
  75   .node_name = "nat44-out2in",
  76   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  77                                "ip4-dhcp-client-detect"),
  78 };
  79 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat44-classify",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  83 };
  84 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  85   .arc_name = "ip4-unicast",
  86   .node_name = "nat44-det-in2out",
  87   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  88 };
  89 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-det-out2in",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  93                                "ip4-dhcp-client-detect"),
  94 };
  95 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  96   .arc_name = "ip4-unicast",
  97   .node_name = "nat44-det-classify",
  98   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  99 };
 100 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-ed-in2out",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 104 };
 105 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 106   .arc_name = "ip4-unicast",
 107   .node_name = "nat44-ed-out2in",
 108   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 109                                "ip4-dhcp-client-detect"),
 110 };
 111 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 112   .arc_name = "ip4-unicast",
 113   .node_name = "nat44-ed-classify",
 114   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 115 };
 116 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 117   .arc_name = "ip4-unicast",
 118   .node_name = "nat44-handoff-classify",
 119   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 120 };
 121 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 122   .arc_name = "ip4-unicast",
 123   .node_name = "nat44-in2out-fast",
 124   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 125 };
 126 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 127   .arc_name = "ip4-unicast",
 128   .node_name = "nat44-out2in-fast",
 129   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 130                                "ip4-dhcp-client-detect"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 133   .arc_name = "ip4-unicast",
 134   .node_name = "nat44-hairpin-dst",
 135   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 136 };
 137 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 138   .arc_name = "ip4-unicast",
 139   .node_name = "nat44-ed-hairpin-dst",
 140   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 141 };
 142
 143 /* Hook up output features */
 144 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 145   .arc_name = "ip4-output",
 146   .node_name = "nat44-in2out-output",
 147   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 148 };
 149 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 150   .arc_name = "ip4-output",
 151   .node_name = "nat44-in2out-output-worker-handoff",
 152   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 153 };
 154 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 155   .arc_name = "ip4-output",
 156   .node_name = "nat44-hairpin-src",
 157   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 158 };
 159 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 160   .arc_name = "ip4-output",
 161   .node_name = "nat44-ed-in2out-output",
 162   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 163 };
 164 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 165   .arc_name = "ip4-output",
 166   .node_name = "nat44-ed-hairpin-src",
 167   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 168 };
 169
 170 /* Hook up ip4-local features */
 171 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 172 {
 173   .arc_name = "ip4-local",
 174   .node_name = "nat44-hairpinning",
 175   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 176 };
 177 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 178 {
 179   .arc_name = "ip4-local",
 180   .node_name = "nat44-ed-hairpinning",
 181   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 182 };
 183
 184
 185 VLIB_PLUGIN_REGISTER () = {
 186     .version = VPP_BUILD_VER,
 187     .description = "Network Address Translation (NAT)",
 188 };
 189 /* *INDENT-ON* */
 190
 191 void
 192 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 193                        u8 is_ha)
 194 {
 195   snat_session_key_t key;
 196   clib_bihash_kv_8_8_t kv;
 197   nat_ed_ses_key_t ed_key;
 198   clib_bihash_kv_16_8_t ed_kv;
 199   snat_main_per_thread_data_t *tsm =
 200     vec_elt_at_index (sm->per_thread_data, thread_index);
 201
 202   if (is_fwd_bypass_session (s))
 203     {
 204       if (snat_is_unk_proto_session (s))
 205         {
 206           ed_key.proto = s->in2out.port;
 207           ed_key.r_port = 0;
 208           ed_key.l_port = 0;
 209         }
 210       else
 211         {
 212           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 213           ed_key.l_port = s->in2out.port;
 214           ed_key.r_port = s->ext_host_port;
 215         }
 216       ed_key.l_addr = s->in2out.addr;
 217       ed_key.r_addr = s->ext_host_addr;
 218       ed_key.fib_index = 0;
 219       ed_kv.key[0] = ed_key.as_u64[0];
 220       ed_kv.key[1] = ed_key.as_u64[1];
 221       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 222         nat_elog_warn ("in2out_ed key del failed");
 223       return;
 224     }
 225
 226   /* session lookup tables */
 227   if (is_ed_session (s))
 228     {
 229       if (is_affinity_sessions (s))
 230         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 231                              s->in2out.protocol, s->out2in.port);
 232       ed_key.l_addr = s->out2in.addr;
 233       ed_key.r_addr = s->ext_host_addr;
 234       ed_key.fib_index = s->out2in.fib_index;
 235       if (snat_is_unk_proto_session (s))
 236         {
 237           ed_key.proto = s->in2out.port;
 238           ed_key.r_port = 0;
 239           ed_key.l_port = 0;
 240         }
 241       else
 242         {
 243           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 244           ed_key.l_port = s->out2in.port;
 245           ed_key.r_port = s->ext_host_port;
 246         }
 247       ed_kv.key[0] = ed_key.as_u64[0];
 248       ed_kv.key[1] = ed_key.as_u64[1];
 249       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 250         nat_elog_warn ("out2in_ed key del failed");
 251       ed_key.l_addr = s->in2out.addr;
 252       ed_key.fib_index = s->in2out.fib_index;
 253       if (!snat_is_unk_proto_session (s))
 254         ed_key.l_port = s->in2out.port;
 255       if (is_twice_nat_session (s))
 256         {
 257           ed_key.r_addr = s->ext_host_nat_addr;
 258           ed_key.r_port = s->ext_host_nat_port;
 259         }
 260       ed_kv.key[0] = ed_key.as_u64[0];
 261       ed_kv.key[1] = ed_key.as_u64[1];
 262       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 263         nat_elog_warn ("in2out_ed key del failed");
 264
 265       if (!is_ha)
 266         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 267                                &s->in2out.addr, s->in2out.port,
 268                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 269                                &s->out2in.addr, s->out2in.port,
 270                                &s->ext_host_addr, s->ext_host_port,
 271                                s->in2out.protocol, is_twice_nat_session (s));
 272     }
 273   else
 274     {
 275       kv.key = s->in2out.as_u64;
 276       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 277         nat_elog_warn ("in2out key del failed");
 278       kv.key = s->out2in.as_u64;
 279       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 280         nat_elog_warn ("out2in key del failed");
 281
 282       if (!is_ha)
 283         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 284                                  &s->in2out.addr, s->in2out.port,
 285                                  &s->out2in.addr, s->out2in.port,
 286                                  s->in2out.protocol);
 287     }
 288
 289   if (snat_is_unk_proto_session (s))
 290     return;
 291
 292   if (!is_ha)
 293     {
 294       /* log NAT event */
 295       snat_ipfix_logging_nat44_ses_delete (thread_index,
 296                                            s->in2out.addr.as_u32,
 297                                            s->out2in.addr.as_u32,
 298                                            s->in2out.protocol,
 299                                            s->in2out.port,
 300                                            s->out2in.port,
 301                                            s->in2out.fib_index);
 302
 303       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 304                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 305                    thread_index);
 306     }
 307
 308   /* Twice NAT address and port for external host */
 309   if (is_twice_nat_session (s))
 310     {
 311       key.protocol = s->in2out.protocol;
 312       key.port = s->ext_host_nat_port;
 313       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 314       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 315                                           thread_index, &key);
 316     }
 317
 318   if (snat_is_session_static (s))
 319     return;
 320
 321   snat_free_outside_address_and_port (sm->addresses, thread_index,
 322                                       &s->out2in);
 323 }
 324
 325 snat_user_t *
 326 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 327                         u32 thread_index)
 328 {
 329   snat_user_t *u = 0;
 330   snat_user_key_t user_key;
 331   clib_bihash_kv_8_8_t kv, value;
 332   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 333   dlist_elt_t *per_user_list_head_elt;
 334
 335   user_key.addr.as_u32 = addr->as_u32;
 336   user_key.fib_index = fib_index;
 337   kv.key = user_key.as_u64;
 338
 339   /* Ever heard of the "user" = src ip4 address before? */
 340   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 341     {
 342       /* no, make a new one */
 343       pool_get (tsm->users, u);
 344       clib_memset (u, 0, sizeof (*u));
 345       u->addr.as_u32 = addr->as_u32;
 346       u->fib_index = fib_index;
 347
 348       pool_get (tsm->list_pool, per_user_list_head_elt);
 349
 350       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 351         tsm->list_pool;
 352
 353       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 354
 355       kv.value = u - tsm->users;
 356
 357       /* add user */
 358       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 359         nat_elog_warn ("user_hash keay add failed");
 360
 361       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 362                                pool_elts (tsm->users));
 363     }
 364   else
 365     {
 366       u = pool_elt_at_index (tsm->users, value.value);
 367     }
 368
 369   return u;
 370 }
 371
 372 snat_session_t *
 373 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 374                               u32 thread_index, f64 now)
 375 {
 376   snat_session_t *s;
 377   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 378   u32 oldest_per_user_translation_list_index, session_index;
 379   dlist_elt_t *oldest_per_user_translation_list_elt;
 380   dlist_elt_t *per_user_translation_list_elt;
 381
 382   /* Over quota? Recycle the least recently used translation */
 383   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 384     {
 385       oldest_per_user_translation_list_index =
 386         clib_dlist_remove_head (tsm->list_pool,
 387                                 u->sessions_per_user_list_head_index);
 388
 389       ASSERT (oldest_per_user_translation_list_index != ~0);
 390
 391       /* Add it back to the end of the LRU list */
 392       clib_dlist_addtail (tsm->list_pool,
 393                           u->sessions_per_user_list_head_index,
 394                           oldest_per_user_translation_list_index);
 395       /* Get the list element */
 396       oldest_per_user_translation_list_elt =
 397         pool_elt_at_index (tsm->list_pool,
 398                            oldest_per_user_translation_list_index);
 399
 400       /* Get the session index from the list element */
 401       session_index = oldest_per_user_translation_list_elt->value;
 402
 403       /* Get the session */
 404       s = pool_elt_at_index (tsm->sessions, session_index);
 405       nat_free_session_data (sm, s, thread_index, 0);
 406       if (snat_is_session_static (s))
 407         u->nstaticsessions--;
 408       else
 409         u->nsessions--;
 410       s->flags = 0;
 411       s->total_bytes = 0;
 412       s->total_pkts = 0;
 413       s->state = 0;
 414       s->ext_host_addr.as_u32 = 0;
 415       s->ext_host_port = 0;
 416       s->ext_host_nat_addr.as_u32 = 0;
 417       s->ext_host_nat_port = 0;
 418     }
 419   else
 420     {
 421       pool_get (tsm->sessions, s);
 422       clib_memset (s, 0, sizeof (*s));
 423
 424       /* Create list elts */
 425       pool_get (tsm->list_pool, per_user_translation_list_elt);
 426       clib_dlist_init (tsm->list_pool,
 427                        per_user_translation_list_elt - tsm->list_pool);
 428
 429       per_user_translation_list_elt->value = s - tsm->sessions;
 430       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 431       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 432
 433       clib_dlist_addtail (tsm->list_pool,
 434                           s->per_user_list_head_index,
 435                           per_user_translation_list_elt - tsm->list_pool);
 436
 437       s->user_index = u - tsm->users;
 438       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 439                                pool_elts (tsm->sessions));
 440     }
 441
 442   s->ha_last_refreshed = now;
 443
 444   return s;
 445 }
 446
 447 snat_session_t *
 448 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 449                       f64 now)
 450 {
 451   snat_session_t *s;
 452   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 453   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 454   u32 oldest_index;
 455   u64 sess_timeout_time;
 456
 457   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 458     goto alloc_new;
 459
 460   oldest_index =
 461     clib_dlist_remove_head (tsm->list_pool,
 462                             u->sessions_per_user_list_head_index);
 463   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 464   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 465
 466   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 467   if (now >= sess_timeout_time)
 468     {
 469       clib_dlist_addtail (tsm->list_pool,
 470                           u->sessions_per_user_list_head_index, oldest_index);
 471       nat_free_session_data (sm, s, thread_index, 0);
 472       if (snat_is_session_static (s))
 473         u->nstaticsessions--;
 474       else
 475         u->nsessions--;
 476       s->flags = 0;
 477       s->total_bytes = 0;
 478       s->total_pkts = 0;
 479       s->state = 0;
 480       s->ext_host_addr.as_u32 = 0;
 481       s->ext_host_port = 0;
 482       s->ext_host_nat_addr.as_u32 = 0;
 483       s->ext_host_nat_port = 0;
 484     }
 485   else
 486     {
 487       clib_dlist_addhead (tsm->list_pool,
 488                           u->sessions_per_user_list_head_index, oldest_index);
 489       if ((u->nsessions + u->nstaticsessions) >=
 490           sm->max_translations_per_user)
 491         {
 492           nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 493                          clib_net_to_host_u32 (u->addr.as_u32));
 494           snat_ipfix_logging_max_entries_per_user
 495             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 496           return 0;
 497         }
 498       else
 499         {
 500         alloc_new:
 501           pool_get (tsm->sessions, s);
 502           clib_memset (s, 0, sizeof (*s));
 503
 504           /* Create list elts */
 505           pool_get (tsm->list_pool, per_user_translation_list_elt);
 506           clib_dlist_init (tsm->list_pool,
 507                            per_user_translation_list_elt - tsm->list_pool);
 508
 509           per_user_translation_list_elt->value = s - tsm->sessions;
 510           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 511           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 512
 513           clib_dlist_addtail (tsm->list_pool,
 514                               s->per_user_list_head_index,
 515                               per_user_translation_list_elt - tsm->list_pool);
 516         }
 517
 518       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 519                                pool_elts (tsm->sessions));
 520     }
 521
 522   s->ha_last_refreshed = now;
 523
 524   return s;
 525 }
 526
 527 void
 528 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 529                           int is_add)
 530 {
 531   fib_prefix_t prefix = {
 532     .fp_len = p_len,
 533     .fp_proto = FIB_PROTOCOL_IP4,
 534     .fp_addr = {
 535                 .ip4.as_u32 = addr->as_u32,
 536                 },
 537   };
 538   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 539
 540   if (is_add)
 541     fib_table_entry_update_one_path (fib_index,
 542                                      &prefix,
 543                                      nat_fib_src_low,
 544                                      (FIB_ENTRY_FLAG_CONNECTED |
 545                                       FIB_ENTRY_FLAG_LOCAL |
 546                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 547                                      DPO_PROTO_IP4,
 548                                      NULL,
 549                                      sw_if_index,
 550                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 551   else
 552     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 553 }
 554
 555 int
 556 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 557                   u8 twice_nat)
 558 {
 559   snat_address_t *ap;
 560   snat_interface_t *i;
 561   vlib_thread_main_t *tm = vlib_get_thread_main ();
 562
 563   if (twice_nat && !sm->endpoint_dependent)
 564     return VNET_API_ERROR_FEATURE_DISABLED;
 565
 566   /* Check if address already exists */
 567   /* *INDENT-OFF* */
 568   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 569     {
 570       if (ap->addr.as_u32 == addr->as_u32)
 571         return VNET_API_ERROR_VALUE_EXIST;
 572     }
 573   /* *INDENT-ON* */
 574
 575   if (twice_nat)
 576     vec_add2 (sm->twice_nat_addresses, ap, 1);
 577   else
 578     vec_add2 (sm->addresses, ap, 1);
 579
 580   ap->addr = *addr;
 581   if (vrf_id != ~0)
 582     ap->fib_index =
 583       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 584                                          nat_fib_src_low);
 585   else
 586     ap->fib_index = ~0;
 587 #define _(N, i, n, s) \
 588   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 589   ap->busy_##n##_ports = 0; \
 590   ap->busy_##n##_ports_per_thread = 0;\
 591   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 592   foreach_snat_protocol
 593 #undef _
 594     if (twice_nat)
 595     return 0;
 596
 597   /* Add external address to FIB */
 598   /* *INDENT-OFF* */
 599   pool_foreach (i, sm->interfaces,
 600   ({
 601     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 602       continue;
 603
 604     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 605     break;
 606   }));
 607   pool_foreach (i, sm->output_feature_interfaces,
 608   ({
 609     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 610       continue;
 611
 612     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 613     break;
 614   }));
 615   /* *INDENT-ON* */
 616
 617   return 0;
 618 }
 619
 620 static int
 621 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 622 {
 623   snat_static_mapping_t *m;
 624   /* *INDENT-OFF* */
 625   pool_foreach (m, sm->static_mappings,
 626   ({
 627       if (is_addr_only_static_mapping (m) ||
 628           is_out2in_only_static_mapping (m) ||
 629           is_identity_static_mapping (m))
 630         continue;
 631       if (m->external_addr.as_u32 == addr.as_u32)
 632         return 1;
 633   }));
 634   /* *INDENT-ON* */
 635
 636   return 0;
 637 }
 638
 639 void
 640 increment_v4_address (ip4_address_t * a)
 641 {
 642   u32 v;
 643
 644   v = clib_net_to_host_u32 (a->as_u32) + 1;
 645   a->as_u32 = clib_host_to_net_u32 (v);
 646 }
 647
 648 static void
 649 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 650                                        ip4_address_t l_addr,
 651                                        u16 l_port,
 652                                        u32 sw_if_index,
 653                                        u16 e_port,
 654                                        u32 vrf_id,
 655                                        snat_protocol_t proto,
 656                                        int addr_only, int is_add, u8 * tag,
 657                                        int twice_nat, int out2in_only,
 658                                        int identity_nat)
 659 {
 660   snat_static_map_resolve_t *rp;
 661
 662   vec_add2 (sm->to_resolve, rp, 1);
 663   rp->l_addr.as_u32 = l_addr.as_u32;
 664   rp->l_port = l_port;
 665   rp->sw_if_index = sw_if_index;
 666   rp->e_port = e_port;
 667   rp->vrf_id = vrf_id;
 668   rp->proto = proto;
 669   rp->addr_only = addr_only;
 670   rp->is_add = is_add;
 671   rp->twice_nat = twice_nat;
 672   rp->out2in_only = out2in_only;
 673   rp->identity_nat = identity_nat;
 674   rp->tag = vec_dup (tag);
 675 }
 676
 677 static u32
 678 get_thread_idx_by_port (u16 e_port)
 679 {
 680   snat_main_t *sm = &snat_main;
 681   u32 thread_idx = sm->num_workers;
 682   if (sm->num_workers > 1)
 683     {
 684       thread_idx =
 685         sm->first_worker_index +
 686         sm->workers[(e_port - 1024) / sm->port_per_thread];
 687     }
 688   return thread_idx;
 689 }
 690
 691 int
 692 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 693                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 694                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 695                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 696                          u8 identity_nat)
 697 {
 698   snat_main_t *sm = &snat_main;
 699   snat_static_mapping_t *m;
 700   snat_session_key_t m_key;
 701   clib_bihash_kv_8_8_t kv, value;
 702   snat_address_t *a = 0;
 703   u32 fib_index = ~0;
 704   snat_interface_t *interface;
 705   int i;
 706   snat_main_per_thread_data_t *tsm;
 707   snat_user_key_t u_key;
 708   snat_user_t *u;
 709   dlist_elt_t *head, *elt;
 710   u32 elt_index, head_index;
 711   u32 ses_index;
 712   u64 user_index;
 713   snat_session_t *s;
 714   snat_static_map_resolve_t *rp, *rp_match = 0;
 715   nat44_lb_addr_port_t *local;
 716   u32 find = ~0;
 717
 718   if (!sm->endpoint_dependent)
 719     {
 720       if (twice_nat || out2in_only)
 721         return VNET_API_ERROR_FEATURE_DISABLED;
 722     }
 723
 724   /* If the external address is a specific interface address */
 725   if (sw_if_index != ~0)
 726     {
 727       ip4_address_t *first_int_addr;
 728
 729       for (i = 0; i < vec_len (sm->to_resolve); i++)
 730         {
 731           rp = sm->to_resolve + i;
 732           if (rp->sw_if_index != sw_if_index ||
 733               rp->l_addr.as_u32 != l_addr.as_u32 ||
 734               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 735             continue;
 736
 737           if (!addr_only)
 738             {
 739               if ((rp->l_port != l_port && rp->e_port != e_port)
 740                   || rp->proto != proto)
 741                 continue;
 742             }
 743
 744           rp_match = rp;
 745           break;
 746         }
 747
 748       /* Might be already set... */
 749       first_int_addr = ip4_interface_first_address
 750         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 751
 752       if (is_add)
 753         {
 754           if (rp_match)
 755             return VNET_API_ERROR_VALUE_EXIST;
 756
 757           snat_add_static_mapping_when_resolved
 758             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 759              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 760
 761           /* DHCP resolution required? */
 762           if (first_int_addr == 0)
 763             {
 764               return 0;
 765             }
 766           else
 767             {
 768               e_addr.as_u32 = first_int_addr->as_u32;
 769               /* Identity mapping? */
 770               if (l_addr.as_u32 == 0)
 771                 l_addr.as_u32 = e_addr.as_u32;
 772             }
 773         }
 774       else
 775         {
 776           if (!rp_match)
 777             return VNET_API_ERROR_NO_SUCH_ENTRY;
 778
 779           vec_del1 (sm->to_resolve, i);
 780
 781           if (first_int_addr)
 782             {
 783               e_addr.as_u32 = first_int_addr->as_u32;
 784               /* Identity mapping? */
 785               if (l_addr.as_u32 == 0)
 786                 l_addr.as_u32 = e_addr.as_u32;
 787             }
 788           else
 789             return 0;
 790         }
 791     }
 792
 793   m_key.addr = e_addr;
 794   m_key.port = addr_only ? 0 : e_port;
 795   m_key.protocol = addr_only ? 0 : proto;
 796   m_key.fib_index = 0;
 797   kv.key = m_key.as_u64;
 798   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 799     m = 0;
 800   else
 801     m = pool_elt_at_index (sm->static_mappings, value.value);
 802
 803   if (is_add)
 804     {
 805       if (m)
 806         {
 807           if (is_identity_static_mapping (m))
 808             {
 809               /* *INDENT-OFF* */
 810               pool_foreach (local, m->locals,
 811               ({
 812                 if (local->vrf_id == vrf_id)
 813                   return VNET_API_ERROR_VALUE_EXIST;
 814               }));
 815               /* *INDENT-ON* */
 816               pool_get (m->locals, local);
 817               local->vrf_id = vrf_id;
 818               local->fib_index =
 819                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 820                                                    nat_fib_src_low);
 821               m_key.addr = m->local_addr;
 822               m_key.port = m->local_port;
 823               m_key.protocol = m->proto;
 824               m_key.fib_index = local->fib_index;
 825               kv.key = m_key.as_u64;
 826               kv.value = m - sm->static_mappings;
 827               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 828               return 0;
 829             }
 830           else
 831             return VNET_API_ERROR_VALUE_EXIST;
 832         }
 833
 834       if (twice_nat && addr_only)
 835         return VNET_API_ERROR_UNSUPPORTED;
 836
 837       /* Convert VRF id to FIB index */
 838       if (vrf_id != ~0)
 839         fib_index =
 840           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 841                                              nat_fib_src_low);
 842       /* If not specified use inside VRF id from SNAT plugin startup config */
 843       else
 844         {
 845           fib_index = sm->inside_fib_index;
 846           vrf_id = sm->inside_vrf_id;
 847           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 848         }
 849
 850       if (!(out2in_only || identity_nat))
 851         {
 852           m_key.addr = l_addr;
 853           m_key.port = addr_only ? 0 : l_port;
 854           m_key.protocol = addr_only ? 0 : proto;
 855           m_key.fib_index = fib_index;
 856           kv.key = m_key.as_u64;
 857           if (!clib_bihash_search_8_8
 858               (&sm->static_mapping_by_local, &kv, &value))
 859             return VNET_API_ERROR_VALUE_EXIST;
 860         }
 861
 862       /* Find external address in allocated addresses and reserve port for
 863          address and port pair mapping when dynamic translations enabled */
 864       if (!(addr_only || sm->static_mapping_only || out2in_only))
 865         {
 866           for (i = 0; i < vec_len (sm->addresses); i++)
 867             {
 868               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 869                 {
 870                   a = sm->addresses + i;
 871                   /* External port must be unused */
 872                   switch (proto)
 873                     {
 874 #define _(N, j, n, s) \
 875                     case SNAT_PROTOCOL_##N: \
 876                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 877                         return VNET_API_ERROR_INVALID_VALUE; \
 878                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 879                       if (e_port > 1024) \
 880                         { \
 881                           a->busy_##n##_ports++; \
 882                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 883                         } \
 884                       break;
 885                       foreach_snat_protocol
 886 #undef _
 887                     default:
 888                       nat_elog_info ("unknown protocol");
 889                       return VNET_API_ERROR_INVALID_VALUE_2;
 890                     }
 891                   break;
 892                 }
 893             }
 894           /* External address must be allocated */
 895           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 896             {
 897               if (sw_if_index != ~0)
 898                 {
 899                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 900                     {
 901                       rp = sm->to_resolve + i;
 902                       if (rp->addr_only)
 903                         continue;
 904                       if (rp->sw_if_index != sw_if_index &&
 905                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 906                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 907                           rp->e_port != e_port && rp->proto != proto)
 908                         continue;
 909
 910                       vec_del1 (sm->to_resolve, i);
 911                       break;
 912                     }
 913                 }
 914               return VNET_API_ERROR_NO_SUCH_ENTRY;
 915             }
 916         }
 917
 918       pool_get (sm->static_mappings, m);
 919       clib_memset (m, 0, sizeof (*m));
 920       m->tag = vec_dup (tag);
 921       m->local_addr = l_addr;
 922       m->external_addr = e_addr;
 923       m->twice_nat = twice_nat;
 924       if (out2in_only)
 925         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 926       if (addr_only)
 927         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 928       if (identity_nat)
 929         {
 930           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 931           pool_get (m->locals, local);
 932           local->vrf_id = vrf_id;
 933           local->fib_index = fib_index;
 934         }
 935       else
 936         {
 937           m->vrf_id = vrf_id;
 938           m->fib_index = fib_index;
 939         }
 940       if (!addr_only)
 941         {
 942           m->local_port = l_port;
 943           m->external_port = e_port;
 944           m->proto = proto;
 945         }
 946
 947       if (sm->num_workers > 1)
 948         {
 949           ip4_header_t ip = {
 950             .src_address = m->local_addr,
 951           };
 952           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 953           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 954         }
 955       else
 956         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 957
 958       m_key.addr = m->local_addr;
 959       m_key.port = m->local_port;
 960       m_key.protocol = m->proto;
 961       m_key.fib_index = fib_index;
 962       kv.key = m_key.as_u64;
 963       kv.value = m - sm->static_mappings;
 964       if (!out2in_only)
 965         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 966
 967       m_key.addr = m->external_addr;
 968       m_key.port = m->external_port;
 969       m_key.fib_index = 0;
 970       kv.key = m_key.as_u64;
 971       kv.value = m - sm->static_mappings;
 972       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 973
 974       /* Delete dynamic sessions matching local address (+ local port) */
 975       if (!(sm->static_mapping_only))
 976         {
 977           u_key.addr = m->local_addr;
 978           u_key.fib_index = m->fib_index;
 979           kv.key = u_key.as_u64;
 980           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 981             {
 982               user_index = value.value;
 983               u = pool_elt_at_index (tsm->users, user_index);
 984               if (u->nsessions)
 985                 {
 986                   head_index = u->sessions_per_user_list_head_index;
 987                   head = pool_elt_at_index (tsm->list_pool, head_index);
 988                   elt_index = head->next;
 989                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 990                   ses_index = elt->value;
 991                   while (ses_index != ~0)
 992                     {
 993                       s = pool_elt_at_index (tsm->sessions, ses_index);
 994                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 995                       ses_index = elt->value;
 996
 997                       if (snat_is_session_static (s))
 998                         continue;
 999
1000                       if (!addr_only
1001                           && (clib_net_to_host_u16 (s->in2out.port) !=
1002                               m->local_port))
1003                         continue;
1004
1005                       nat_free_session_data (sm, s,
1006                                              tsm - sm->per_thread_data, 0);
1007                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1008
1009                       if (!addr_only && !sm->endpoint_dependent)
1010                         break;
1011                     }
1012                 }
1013             }
1014         }
1015     }
1016   else
1017     {
1018       if (!m)
1019         {
1020           if (sw_if_index != ~0)
1021             return 0;
1022           else
1023             return VNET_API_ERROR_NO_SUCH_ENTRY;
1024         }
1025
1026       if (identity_nat)
1027         {
1028           if (vrf_id == ~0)
1029             vrf_id = sm->inside_vrf_id;
1030
1031           /* *INDENT-OFF* */
1032           pool_foreach (local, m->locals,
1033           ({
1034             if (local->vrf_id == vrf_id)
1035               find = local - m->locals;
1036           }));
1037           /* *INDENT-ON* */
1038           if (find == ~0)
1039             return VNET_API_ERROR_NO_SUCH_ENTRY;
1040
1041           local = pool_elt_at_index (m->locals, find);
1042           fib_index = local->fib_index;
1043           pool_put (m->locals, local);
1044         }
1045       else
1046         fib_index = m->fib_index;
1047
1048       /* Free external address port */
1049       if (!(addr_only || sm->static_mapping_only || out2in_only))
1050         {
1051           for (i = 0; i < vec_len (sm->addresses); i++)
1052             {
1053               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1054                 {
1055                   a = sm->addresses + i;
1056                   switch (proto)
1057                     {
1058 #define _(N, j, n, s) \
1059                     case SNAT_PROTOCOL_##N: \
1060                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1061                       if (e_port > 1024) \
1062                         { \
1063                           a->busy_##n##_ports--; \
1064                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1065                         } \
1066                       break;
1067                       foreach_snat_protocol
1068 #undef _
1069                     default:
1070                       nat_elog_info ("unknown protocol");
1071                       return VNET_API_ERROR_INVALID_VALUE_2;
1072                     }
1073                   break;
1074                 }
1075             }
1076         }
1077
1078       if (sm->num_workers > 1)
1079         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1080       else
1081         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1082
1083       m_key.addr = m->local_addr;
1084       m_key.port = m->local_port;
1085       m_key.protocol = m->proto;
1086       m_key.fib_index = fib_index;
1087       kv.key = m_key.as_u64;
1088       if (!out2in_only)
1089         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1090
1091       /* Delete session(s) for static mapping if exist */
1092       if (!(sm->static_mapping_only) ||
1093           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1094         {
1095           u_key.addr = m->local_addr;
1096           u_key.fib_index = fib_index;
1097           kv.key = u_key.as_u64;
1098           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1099             {
1100               user_index = value.value;
1101               u = pool_elt_at_index (tsm->users, user_index);
1102               if (u->nstaticsessions)
1103                 {
1104                   head_index = u->sessions_per_user_list_head_index;
1105                   head = pool_elt_at_index (tsm->list_pool, head_index);
1106                   elt_index = head->next;
1107                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1108                   ses_index = elt->value;
1109                   while (ses_index != ~0)
1110                     {
1111                       s = pool_elt_at_index (tsm->sessions, ses_index);
1112                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1113                       ses_index = elt->value;
1114
1115                       if (!addr_only)
1116                         {
1117                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1118                               (clib_net_to_host_u16 (s->out2in.port) !=
1119                                e_port))
1120                             continue;
1121                         }
1122
1123                       if (is_lb_session (s))
1124                         continue;
1125
1126                       if (!snat_is_session_static (s))
1127                         continue;
1128
1129                       nat_free_session_data (sm, s,
1130                                              tsm - sm->per_thread_data, 0);
1131                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1132
1133                       if (!addr_only && !sm->endpoint_dependent)
1134                         break;
1135                     }
1136                 }
1137             }
1138         }
1139
1140       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1141       if (pool_elts (m->locals))
1142         return 0;
1143
1144       m_key.addr = m->external_addr;
1145       m_key.port = m->external_port;
1146       m_key.fib_index = 0;
1147       kv.key = m_key.as_u64;
1148       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1149
1150       vec_free (m->tag);
1151       vec_free (m->workers);
1152       /* Delete static mapping from pool */
1153       pool_put (sm->static_mappings, m);
1154     }
1155
1156   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1157     return 0;
1158
1159   /* Add/delete external address to FIB */
1160   /* *INDENT-OFF* */
1161   pool_foreach (interface, sm->interfaces,
1162   ({
1163     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1164       continue;
1165
1166     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1167     break;
1168   }));
1169   pool_foreach (interface, sm->output_feature_interfaces,
1170   ({
1171     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1172       continue;
1173
1174     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1175     break;
1176   }));
1177   /* *INDENT-ON* */
1178
1179   return 0;
1180 }
1181
1182 int
1183 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1184                                  snat_protocol_t proto,
1185                                  nat44_lb_addr_port_t * locals, u8 is_add,
1186                                  twice_nat_type_t twice_nat, u8 out2in_only,
1187                                  u8 * tag, u32 affinity)
1188 {
1189   snat_main_t *sm = &snat_main;
1190   snat_static_mapping_t *m;
1191   snat_session_key_t m_key;
1192   clib_bihash_kv_8_8_t kv, value;
1193   snat_address_t *a = 0;
1194   int i;
1195   nat44_lb_addr_port_t *local;
1196   u32 elt_index, head_index, ses_index;
1197   snat_main_per_thread_data_t *tsm;
1198   snat_user_key_t u_key;
1199   snat_user_t *u;
1200   snat_session_t *s;
1201   dlist_elt_t *head, *elt;
1202   uword *bitmap = 0;
1203
1204   if (!sm->endpoint_dependent)
1205     return VNET_API_ERROR_FEATURE_DISABLED;
1206
1207   m_key.addr = e_addr;
1208   m_key.port = e_port;
1209   m_key.protocol = proto;
1210   m_key.fib_index = 0;
1211   kv.key = m_key.as_u64;
1212   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1213     m = 0;
1214   else
1215     m = pool_elt_at_index (sm->static_mappings, value.value);
1216
1217   if (is_add)
1218     {
1219       if (m)
1220         return VNET_API_ERROR_VALUE_EXIST;
1221
1222       if (vec_len (locals) < 2)
1223         return VNET_API_ERROR_INVALID_VALUE;
1224
1225       /* Find external address in allocated addresses and reserve port for
1226          address and port pair mapping when dynamic translations enabled */
1227       if (!(sm->static_mapping_only || out2in_only))
1228         {
1229           for (i = 0; i < vec_len (sm->addresses); i++)
1230             {
1231               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1232                 {
1233                   a = sm->addresses + i;
1234                   /* External port must be unused */
1235                   switch (proto)
1236                     {
1237 #define _(N, j, n, s) \
1238                     case SNAT_PROTOCOL_##N: \
1239                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1240                         return VNET_API_ERROR_INVALID_VALUE; \
1241                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1242                       if (e_port > 1024) \
1243                         { \
1244                           a->busy_##n##_ports++; \
1245                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1246                         } \
1247                       break;
1248                       foreach_snat_protocol
1249 #undef _
1250                     default:
1251                       nat_elog_info ("unknown protocol");
1252                       return VNET_API_ERROR_INVALID_VALUE_2;
1253                     }
1254                   break;
1255                 }
1256             }
1257           /* External address must be allocated */
1258           if (!a)
1259             return VNET_API_ERROR_NO_SUCH_ENTRY;
1260         }
1261
1262       pool_get (sm->static_mappings, m);
1263       clib_memset (m, 0, sizeof (*m));
1264       m->tag = vec_dup (tag);
1265       m->external_addr = e_addr;
1266       m->external_port = e_port;
1267       m->proto = proto;
1268       m->twice_nat = twice_nat;
1269       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1270       if (out2in_only)
1271         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1272       m->affinity = affinity;
1273
1274       if (affinity)
1275         m->affinity_per_service_list_head_index =
1276           nat_affinity_get_per_service_list_head_index ();
1277       else
1278         m->affinity_per_service_list_head_index = ~0;
1279
1280       m_key.addr = m->external_addr;
1281       m_key.port = m->external_port;
1282       m_key.protocol = m->proto;
1283       m_key.fib_index = 0;
1284       kv.key = m_key.as_u64;
1285       kv.value = m - sm->static_mappings;
1286       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1287         {
1288           nat_elog_err ("static_mapping_by_external key add failed");
1289           return VNET_API_ERROR_UNSPECIFIED;
1290         }
1291
1292       m_key.fib_index = m->fib_index;
1293       for (i = 0; i < vec_len (locals); i++)
1294         {
1295           locals[i].fib_index =
1296             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1297                                                locals[i].vrf_id,
1298                                                nat_fib_src_low);
1299           m_key.addr = locals[i].addr;
1300           m_key.fib_index = locals[i].fib_index;
1301           if (!out2in_only)
1302             {
1303               m_key.port = locals[i].port;
1304               kv.key = m_key.as_u64;
1305               kv.value = m - sm->static_mappings;
1306               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1307             }
1308           locals[i].prefix = (i == 0) ? locals[i].probability :
1309             (locals[i - 1].prefix + locals[i].probability);
1310           pool_get (m->locals, local);
1311           *local = locals[i];
1312           if (sm->num_workers > 1)
1313             {
1314               ip4_header_t ip = {
1315                 .src_address = locals[i].addr,
1316               };
1317               bitmap =
1318                 clib_bitmap_set (bitmap,
1319                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1320                                  1);
1321             }
1322         }
1323
1324       /* Assign workers */
1325       if (sm->num_workers > 1)
1326         {
1327           /* *INDENT-OFF* */
1328           clib_bitmap_foreach (i, bitmap,
1329             ({
1330                vec_add1(m->workers, i);
1331             }));
1332           /* *INDENT-ON* */
1333         }
1334     }
1335   else
1336     {
1337       if (!m)
1338         return VNET_API_ERROR_NO_SUCH_ENTRY;
1339
1340       if (!is_lb_static_mapping (m))
1341         return VNET_API_ERROR_INVALID_VALUE;
1342
1343       /* Free external address port */
1344       if (!(sm->static_mapping_only || out2in_only))
1345         {
1346           for (i = 0; i < vec_len (sm->addresses); i++)
1347             {
1348               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1349                 {
1350                   a = sm->addresses + i;
1351                   switch (proto)
1352                     {
1353 #define _(N, j, n, s) \
1354                     case SNAT_PROTOCOL_##N: \
1355                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1356                       if (e_port > 1024) \
1357                         { \
1358                           a->busy_##n##_ports--; \
1359                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1360                         } \
1361                       break;
1362                       foreach_snat_protocol
1363 #undef _
1364                     default:
1365                       nat_elog_info ("unknown protocol");
1366                       return VNET_API_ERROR_INVALID_VALUE_2;
1367                     }
1368                   break;
1369                 }
1370             }
1371         }
1372
1373       m_key.addr = m->external_addr;
1374       m_key.port = m->external_port;
1375       m_key.protocol = m->proto;
1376       m_key.fib_index = 0;
1377       kv.key = m_key.as_u64;
1378       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1379         {
1380           nat_elog_err ("static_mapping_by_external key del failed");
1381           return VNET_API_ERROR_UNSPECIFIED;
1382         }
1383
1384       /* *INDENT-OFF* */
1385       pool_foreach (local, m->locals,
1386       ({
1387           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1388                             nat_fib_src_low);
1389           m_key.addr = local->addr;
1390           if (!out2in_only)
1391             {
1392               m_key.port = local->port;
1393               m_key.fib_index = local->fib_index;
1394               kv.key = m_key.as_u64;
1395               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1396                 {
1397                   nat_elog_err ("static_mapping_by_local key del failed");
1398                   return VNET_API_ERROR_UNSPECIFIED;
1399                 }
1400             }
1401
1402           if (sm->num_workers > 1)
1403             {
1404               ip4_header_t ip = {
1405                 .src_address = local->addr,
1406               };
1407               tsm = vec_elt_at_index (sm->per_thread_data,
1408                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1409             }
1410           else
1411             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1412
1413           /* Delete sessions */
1414           u_key.addr = local->addr;
1415           u_key.fib_index = local->fib_index;
1416           kv.key = u_key.as_u64;
1417           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1418             {
1419               u = pool_elt_at_index (tsm->users, value.value);
1420               if (u->nstaticsessions)
1421                 {
1422                   head_index = u->sessions_per_user_list_head_index;
1423                   head = pool_elt_at_index (tsm->list_pool, head_index);
1424                   elt_index = head->next;
1425                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1426                   ses_index = elt->value;
1427                   while (ses_index != ~0)
1428                     {
1429                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1430                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1431                       ses_index = elt->value;
1432
1433                       if (!(is_lb_session (s)))
1434                         continue;
1435
1436                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1437                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1438                         continue;
1439
1440                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1441                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1442                     }
1443                 }
1444             }
1445       }));
1446       /* *INDENT-ON* */
1447       if (m->affinity)
1448         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1449       pool_free (m->locals);
1450       vec_free (m->tag);
1451       vec_free (m->workers);
1452
1453       pool_put (sm->static_mappings, m);
1454     }
1455
1456   return 0;
1457 }
1458
1459 int
1460 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1461                                        ip4_address_t l_addr, u16 l_port,
1462                                        snat_protocol_t proto, u32 vrf_id,
1463                                        u8 probability, u8 is_add)
1464 {
1465   snat_main_t *sm = &snat_main;
1466   snat_static_mapping_t *m = 0;
1467   snat_session_key_t m_key;
1468   clib_bihash_kv_8_8_t kv, value;
1469   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1470   snat_main_per_thread_data_t *tsm;
1471   snat_user_key_t u_key;
1472   snat_user_t *u;
1473   snat_session_t *s;
1474   dlist_elt_t *head, *elt;
1475   u32 elt_index, head_index, ses_index, *locals = 0;
1476   uword *bitmap = 0;
1477   int i;
1478
1479   if (!sm->endpoint_dependent)
1480     return VNET_API_ERROR_FEATURE_DISABLED;
1481
1482   m_key.addr = e_addr;
1483   m_key.port = e_port;
1484   m_key.protocol = proto;
1485   m_key.fib_index = 0;
1486   kv.key = m_key.as_u64;
1487   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1488     m = pool_elt_at_index (sm->static_mappings, value.value);
1489
1490   if (!m)
1491     return VNET_API_ERROR_NO_SUCH_ENTRY;
1492
1493   if (!is_lb_static_mapping (m))
1494     return VNET_API_ERROR_INVALID_VALUE;
1495
1496   /* *INDENT-OFF* */
1497   pool_foreach (local, m->locals,
1498   ({
1499     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1500         (local->vrf_id == vrf_id))
1501       {
1502         match_local = local;
1503         break;
1504       }
1505   }));
1506   /* *INDENT-ON* */
1507
1508   if (is_add)
1509     {
1510       if (match_local)
1511         return VNET_API_ERROR_VALUE_EXIST;
1512
1513       pool_get (m->locals, local);
1514       clib_memset (local, 0, sizeof (*local));
1515       local->addr.as_u32 = l_addr.as_u32;
1516       local->port = l_port;
1517       local->probability = probability;
1518       local->vrf_id = vrf_id;
1519       local->fib_index =
1520         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1521                                            nat_fib_src_low);
1522
1523       if (!is_out2in_only_static_mapping (m))
1524         {
1525           m_key.addr = l_addr;
1526           m_key.port = l_port;
1527           m_key.fib_index = local->fib_index;
1528           kv.key = m_key.as_u64;
1529           kv.value = m - sm->static_mappings;
1530           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1531             nat_elog_err ("static_mapping_by_local key add failed");
1532         }
1533     }
1534   else
1535     {
1536       if (!match_local)
1537         return VNET_API_ERROR_NO_SUCH_ENTRY;
1538
1539       if (pool_elts (m->locals) < 3)
1540         return VNET_API_ERROR_UNSPECIFIED;
1541
1542       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1543                         nat_fib_src_low);
1544
1545       if (!is_out2in_only_static_mapping (m))
1546         {
1547           m_key.addr = l_addr;
1548           m_key.port = l_port;
1549           m_key.fib_index = match_local->fib_index;
1550           kv.key = m_key.as_u64;
1551           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1552             nat_elog_err ("static_mapping_by_local key del failed");
1553         }
1554
1555       if (sm->num_workers > 1)
1556         {
1557           ip4_header_t ip = {
1558             .src_address = local->addr,
1559           };
1560           tsm = vec_elt_at_index (sm->per_thread_data,
1561                                   sm->worker_in2out_cb (&ip, m->fib_index,
1562                                                         0));
1563         }
1564       else
1565         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1566
1567       /* Delete sessions */
1568       u_key.addr = match_local->addr;
1569       u_key.fib_index = match_local->fib_index;
1570       kv.key = u_key.as_u64;
1571       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1572         {
1573           u = pool_elt_at_index (tsm->users, value.value);
1574           if (u->nstaticsessions)
1575             {
1576               head_index = u->sessions_per_user_list_head_index;
1577               head = pool_elt_at_index (tsm->list_pool, head_index);
1578               elt_index = head->next;
1579               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1580               ses_index = elt->value;
1581               while (ses_index != ~0)
1582                 {
1583                   s = pool_elt_at_index (tsm->sessions, ses_index);
1584                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1585                   ses_index = elt->value;
1586
1587                   if (!(is_lb_session (s)))
1588                     continue;
1589
1590                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1591                       (clib_net_to_host_u16 (s->in2out.port) !=
1592                        match_local->port))
1593                     continue;
1594
1595                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1596                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1597                 }
1598             }
1599         }
1600
1601       pool_put (m->locals, match_local);
1602     }
1603
1604   vec_free (m->workers);
1605
1606   /* *INDENT-OFF* */
1607   pool_foreach (local, m->locals,
1608   ({
1609     vec_add1 (locals, local - m->locals);
1610     if (sm->num_workers > 1)
1611       {
1612         ip4_header_t ip;
1613         ip.src_address.as_u32 = local->addr.as_u32,
1614         bitmap = clib_bitmap_set (bitmap,
1615                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1616                                   1);
1617       }
1618   }));
1619   /* *INDENT-ON* */
1620
1621   ASSERT (vec_len (locals) > 1);
1622
1623   local = pool_elt_at_index (m->locals, locals[0]);
1624   local->prefix = local->probability;
1625   for (i = 1; i < vec_len (locals); i++)
1626     {
1627       local = pool_elt_at_index (m->locals, locals[i]);
1628       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1629       local->prefix = local->probability + prev_local->prefix;
1630     }
1631
1632   /* Assign workers */
1633   if (sm->num_workers > 1)
1634     {
1635       /* *INDENT-OFF* */
1636       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1637       /* *INDENT-ON* */
1638     }
1639
1640   return 0;
1641 }
1642
1643 int
1644 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1645                   u8 twice_nat)
1646 {
1647   snat_address_t *a = 0;
1648   snat_session_t *ses;
1649   u32 *ses_to_be_removed = 0, *ses_index;
1650   snat_main_per_thread_data_t *tsm;
1651   snat_static_mapping_t *m;
1652   snat_interface_t *interface;
1653   int i;
1654   snat_address_t *addresses =
1655     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1656
1657   /* Find SNAT address */
1658   for (i = 0; i < vec_len (addresses); i++)
1659     {
1660       if (addresses[i].addr.as_u32 == addr.as_u32)
1661         {
1662           a = addresses + i;
1663           break;
1664         }
1665     }
1666   if (!a)
1667     return VNET_API_ERROR_NO_SUCH_ENTRY;
1668
1669   if (delete_sm)
1670     {
1671       /* *INDENT-OFF* */
1672       pool_foreach (m, sm->static_mappings,
1673       ({
1674           if (m->external_addr.as_u32 == addr.as_u32)
1675             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1676                                             m->local_port, m->external_port,
1677                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1678                                             m->proto, 0, m->twice_nat,
1679                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1680       }));
1681       /* *INDENT-ON* */
1682     }
1683   else
1684     {
1685       /* Check if address is used in some static mapping */
1686       if (is_snat_address_used_in_static_mapping (sm, addr))
1687         {
1688           nat_elog_notice ("address used in static mapping");
1689           return VNET_API_ERROR_UNSPECIFIED;
1690         }
1691     }
1692
1693   if (a->fib_index != ~0)
1694     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1695
1696   /* Delete sessions using address */
1697   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1698     {
1699       /* *INDENT-OFF* */
1700       vec_foreach (tsm, sm->per_thread_data)
1701         {
1702           pool_foreach (ses, tsm->sessions, ({
1703             if (ses->out2in.addr.as_u32 == addr.as_u32)
1704               {
1705                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1706                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1707               }
1708           }));
1709
1710           vec_foreach (ses_index, ses_to_be_removed)
1711             {
1712               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1713               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1714             }
1715
1716           vec_free (ses_to_be_removed);
1717         }
1718       /* *INDENT-ON* */
1719     }
1720
1721 #define _(N, i, n, s) \
1722   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1723   vec_free (a->busy_##n##_ports_per_thread);
1724   foreach_snat_protocol
1725 #undef _
1726     if (twice_nat)
1727     {
1728       vec_del1 (sm->twice_nat_addresses, i);
1729       return 0;
1730     }
1731   else
1732     vec_del1 (sm->addresses, i);
1733
1734   /* Delete external address from FIB */
1735   /* *INDENT-OFF* */
1736   pool_foreach (interface, sm->interfaces,
1737   ({
1738     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1739       continue;
1740
1741     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1742     break;
1743   }));
1744   pool_foreach (interface, sm->output_feature_interfaces,
1745   ({
1746     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1747       continue;
1748
1749     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1750     break;
1751   }));
1752   /* *INDENT-ON* */
1753
1754   return 0;
1755 }
1756
1757 int
1758 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1759 {
1760   snat_main_t *sm = &snat_main;
1761   snat_interface_t *i;
1762   const char *feature_name, *del_feature_name;
1763   snat_address_t *ap;
1764   snat_static_mapping_t *m;
1765   snat_det_map_t *dm;
1766   nat_outside_fib_t *outside_fib;
1767   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1768                                                        sw_if_index);
1769
1770   if (sm->out2in_dpo && !is_inside)
1771     return VNET_API_ERROR_UNSUPPORTED;
1772
1773   /* *INDENT-OFF* */
1774   pool_foreach (i, sm->output_feature_interfaces,
1775   ({
1776     if (i->sw_if_index == sw_if_index)
1777       return VNET_API_ERROR_VALUE_EXIST;
1778   }));
1779   /* *INDENT-ON* */
1780
1781   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1782     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1783   else
1784     {
1785       if (sm->num_workers > 1 && !sm->deterministic)
1786         feature_name =
1787           is_inside ? "nat44-in2out-worker-handoff" :
1788           "nat44-out2in-worker-handoff";
1789       else if (sm->deterministic)
1790         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1791       else if (sm->endpoint_dependent)
1792         {
1793           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1794         }
1795       else
1796         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1797     }
1798
1799   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1800     sm->fq_in2out_index =
1801       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1802
1803   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1804     sm->fq_out2in_index =
1805       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1806
1807   if (!is_inside)
1808     {
1809       /* *INDENT-OFF* */
1810       vec_foreach (outside_fib, sm->outside_fibs)
1811         {
1812           if (outside_fib->fib_index == fib_index)
1813             {
1814               if (is_del)
1815                 {
1816                   outside_fib->refcount--;
1817                   if (!outside_fib->refcount)
1818                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1819                 }
1820               else
1821                 outside_fib->refcount++;
1822               goto feature_set;
1823             }
1824         }
1825       /* *INDENT-ON* */
1826       if (!is_del)
1827         {
1828           vec_add2 (sm->outside_fibs, outside_fib, 1);
1829           outside_fib->refcount = 1;
1830           outside_fib->fib_index = fib_index;
1831         }
1832     }
1833 feature_set:
1834   /* *INDENT-OFF* */
1835   pool_foreach (i, sm->interfaces,
1836   ({
1837     if (i->sw_if_index == sw_if_index)
1838       {
1839         if (is_del)
1840           {
1841             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1842               {
1843                 if (is_inside)
1844                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1845                 else
1846                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1847
1848                 if (sm->num_workers > 1 && !sm->deterministic)
1849                   {
1850                     del_feature_name = "nat44-handoff-classify";
1851                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1852                                                  "nat44-out2in-worker-handoff";
1853                   }
1854                 else if (sm->deterministic)
1855                   {
1856                     del_feature_name = "nat44-det-classify";
1857                     feature_name = !is_inside ?  "nat44-det-in2out" :
1858                                                  "nat44-det-out2in";
1859                   }
1860                 else if (sm->endpoint_dependent)
1861                   {
1862                     del_feature_name = "nat44-ed-classify";
1863                     feature_name = !is_inside ?  "nat-pre-in2out" :
1864                                                  "nat-pre-out2in";
1865                   }
1866                 else
1867                   {
1868                     del_feature_name = "nat44-classify";
1869                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1870                   }
1871
1872                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1873                                              sw_if_index, 0, 0, 0);
1874                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1875                                              sw_if_index, 1, 0, 0);
1876                 if (!is_inside)
1877                   {
1878                     if (sm->endpoint_dependent)
1879                       vnet_feature_enable_disable ("ip4-local",
1880                                                    "nat44-ed-hairpinning",
1881                                                    sw_if_index, 1, 0, 0);
1882                     else if (!sm->deterministic)
1883                       vnet_feature_enable_disable ("ip4-local",
1884                                                    "nat44-hairpinning",
1885                                                    sw_if_index, 1, 0, 0);
1886                   }
1887               }
1888             else
1889               {
1890                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1891                                              sw_if_index, 0, 0, 0);
1892                 pool_put (sm->interfaces, i);
1893                 if (is_inside)
1894                   {
1895                     if (sm->endpoint_dependent)
1896                       vnet_feature_enable_disable ("ip4-local",
1897                                                    "nat44-ed-hairpinning",
1898                                                    sw_if_index, 0, 0, 0);
1899                     else if (!sm->deterministic)
1900                       vnet_feature_enable_disable ("ip4-local",
1901                                                    "nat44-hairpinning",
1902                                                    sw_if_index, 0, 0, 0);
1903                   }
1904               }
1905           }
1906         else
1907           {
1908             if ((nat_interface_is_inside(i) && is_inside) ||
1909                 (nat_interface_is_outside(i) && !is_inside))
1910               return 0;
1911
1912             if (sm->num_workers > 1 && !sm->deterministic)
1913               {
1914                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1915                                                  "nat44-out2in-worker-handoff";
1916                 feature_name = "nat44-handoff-classify";
1917               }
1918             else if (sm->deterministic)
1919               {
1920                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1921                                                  "nat44-det-out2in";
1922                 feature_name = "nat44-det-classify";
1923               }
1924             else if (sm->endpoint_dependent)
1925               {
1926                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
1927                                                  "nat-pre-out2in";
1928
1929                 feature_name = "nat44-ed-classify";
1930               }
1931             else
1932               {
1933                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1934                 feature_name = "nat44-classify";
1935               }
1936
1937             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1938                                          sw_if_index, 0, 0, 0);
1939             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1940                                          sw_if_index, 1, 0, 0);
1941             if (!is_inside)
1942               {
1943                 if (sm->endpoint_dependent)
1944                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1945                                                sw_if_index, 0, 0, 0);
1946                 else if (!sm->deterministic)
1947                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1948                                                sw_if_index, 0, 0, 0);
1949               }
1950             goto set_flags;
1951           }
1952
1953         goto fib;
1954       }
1955   }));
1956   /* *INDENT-ON* */
1957
1958   if (is_del)
1959     return VNET_API_ERROR_NO_SUCH_ENTRY;
1960
1961   pool_get (sm->interfaces, i);
1962   i->sw_if_index = sw_if_index;
1963   i->flags = 0;
1964   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1965                                0);
1966
1967   if (is_inside && !sm->out2in_dpo)
1968     {
1969       if (sm->endpoint_dependent)
1970         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1971                                      sw_if_index, 1, 0, 0);
1972       else if (!sm->deterministic)
1973         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1974                                      sw_if_index, 1, 0, 0);
1975     }
1976
1977 set_flags:
1978   if (is_inside)
1979     {
1980       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1981       return 0;
1982     }
1983   else
1984     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1985
1986   /* Add/delete external addresses to FIB */
1987 fib:
1988   /* *INDENT-OFF* */
1989   vec_foreach (ap, sm->addresses)
1990     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1991
1992   pool_foreach (m, sm->static_mappings,
1993   ({
1994     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1995       continue;
1996
1997     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1998   }));
1999
2000   pool_foreach (dm, sm->det_maps,
2001   ({
2002     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2003   }));
2004   /* *INDENT-ON* */
2005
2006   return 0;
2007 }
2008
2009 int
2010 snat_interface_add_del_output_feature (u32 sw_if_index,
2011                                        u8 is_inside, int is_del)
2012 {
2013   snat_main_t *sm = &snat_main;
2014   snat_interface_t *i;
2015   snat_address_t *ap;
2016   snat_static_mapping_t *m;
2017   nat_outside_fib_t *outside_fib;
2018   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2019                                                        sw_if_index);
2020
2021
2022   if (sm->deterministic ||
2023       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2024     return VNET_API_ERROR_UNSUPPORTED;
2025
2026   /* *INDENT-OFF* */
2027   pool_foreach (i, sm->interfaces,
2028   ({
2029     if (i->sw_if_index == sw_if_index)
2030       return VNET_API_ERROR_VALUE_EXIST;
2031   }));
2032   /* *INDENT-ON* */
2033
2034   if (!is_inside)
2035     {
2036       /* *INDENT-OFF* */
2037       vec_foreach (outside_fib, sm->outside_fibs)
2038         {
2039           if (outside_fib->fib_index == fib_index)
2040             {
2041               if (is_del)
2042                 {
2043                   outside_fib->refcount--;
2044                   if (!outside_fib->refcount)
2045                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2046                 }
2047               else
2048                 outside_fib->refcount++;
2049               goto feature_set;
2050             }
2051         }
2052       /* *INDENT-ON* */
2053       if (!is_del)
2054         {
2055           vec_add2 (sm->outside_fibs, outside_fib, 1);
2056           outside_fib->refcount = 1;
2057           outside_fib->fib_index = fib_index;
2058         }
2059     }
2060
2061 feature_set:
2062   if (is_inside)
2063     {
2064       if (sm->endpoint_dependent)
2065         {
2066           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2067                                        sw_if_index, !is_del, 0, 0);
2068           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2069                                        sw_if_index, !is_del, 0, 0);
2070         }
2071       else
2072         {
2073           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2074                                        sw_if_index, !is_del, 0, 0);
2075           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2076                                        sw_if_index, !is_del, 0, 0);
2077         }
2078       goto fq;
2079     }
2080
2081   if (sm->num_workers > 1)
2082     {
2083       vnet_feature_enable_disable ("ip4-unicast",
2084                                    "nat44-out2in-worker-handoff",
2085                                    sw_if_index, !is_del, 0, 0);
2086       vnet_feature_enable_disable ("ip4-output",
2087                                    "nat44-in2out-output-worker-handoff",
2088                                    sw_if_index, !is_del, 0, 0);
2089     }
2090   else
2091     {
2092       if (sm->endpoint_dependent)
2093         {
2094           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2095                                        sw_if_index, !is_del, 0, 0);
2096           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2097                                        sw_if_index, !is_del, 0, 0);
2098         }
2099       else
2100         {
2101           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2102                                        sw_if_index, !is_del, 0, 0);
2103           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2104                                        sw_if_index, !is_del, 0, 0);
2105         }
2106     }
2107
2108 fq:
2109   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2110     sm->fq_in2out_output_index =
2111       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2112
2113   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2114     sm->fq_out2in_index =
2115       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2116
2117   /* *INDENT-OFF* */
2118   pool_foreach (i, sm->output_feature_interfaces,
2119   ({
2120     if (i->sw_if_index == sw_if_index)
2121       {
2122         if (is_del)
2123           pool_put (sm->output_feature_interfaces, i);
2124         else
2125           return VNET_API_ERROR_VALUE_EXIST;
2126
2127         goto fib;
2128       }
2129   }));
2130   /* *INDENT-ON* */
2131
2132   if (is_del)
2133     return VNET_API_ERROR_NO_SUCH_ENTRY;
2134
2135   pool_get (sm->output_feature_interfaces, i);
2136   i->sw_if_index = sw_if_index;
2137   i->flags = 0;
2138   if (is_inside)
2139     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2140   else
2141     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2142
2143   /* Add/delete external addresses to FIB */
2144 fib:
2145   if (is_inside)
2146     return 0;
2147
2148   /* *INDENT-OFF* */
2149   vec_foreach (ap, sm->addresses)
2150     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2151
2152   pool_foreach (m, sm->static_mappings,
2153   ({
2154     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2155       continue;
2156
2157     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2158   }));
2159   /* *INDENT-ON* */
2160
2161   return 0;
2162 }
2163
2164 int
2165 snat_set_workers (uword * bitmap)
2166 {
2167   snat_main_t *sm = &snat_main;
2168   int i, j = 0;
2169
2170   if (sm->num_workers < 2)
2171     return VNET_API_ERROR_FEATURE_DISABLED;
2172
2173   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2174     return VNET_API_ERROR_INVALID_WORKER;
2175
2176   vec_free (sm->workers);
2177   /* *INDENT-OFF* */
2178   clib_bitmap_foreach (i, bitmap,
2179     ({
2180       vec_add1(sm->workers, i);
2181       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2182       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2183       j++;
2184     }));
2185   /* *INDENT-ON* */
2186
2187   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2188   sm->num_snat_thread = _vec_len (sm->workers);
2189
2190   return 0;
2191 }
2192
2193 static void
2194 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2195                          u32 old_fib_index)
2196 {
2197   snat_main_t *sm = &snat_main;
2198   nat_outside_fib_t *outside_fib;
2199   snat_interface_t *i;
2200   u8 is_add = 1;
2201   u8 match = 0;
2202
2203   if (new_fib_index == old_fib_index)
2204     return;
2205
2206   if (!vec_len (sm->outside_fibs))
2207     return;
2208
2209   /* *INDENT-OFF* */
2210   pool_foreach (i, sm->interfaces,
2211     ({
2212       if (i->sw_if_index == sw_if_index)
2213         {
2214           if (!(nat_interface_is_outside (i)))
2215             return;
2216           match = 1;
2217         }
2218     }));
2219
2220   pool_foreach (i, sm->output_feature_interfaces,
2221     ({
2222       if (i->sw_if_index == sw_if_index)
2223         {
2224           if (!(nat_interface_is_outside (i)))
2225             return;
2226           match = 1;
2227         }
2228     }));
2229   /* *INDENT-ON* */
2230
2231   if (!match)
2232     return;
2233
2234   vec_foreach (outside_fib, sm->outside_fibs)
2235   {
2236     if (outside_fib->fib_index == old_fib_index)
2237       {
2238         outside_fib->refcount--;
2239         if (!outside_fib->refcount)
2240           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2241         break;
2242       }
2243   }
2244
2245   vec_foreach (outside_fib, sm->outside_fibs)
2246   {
2247     if (outside_fib->fib_index == new_fib_index)
2248       {
2249         outside_fib->refcount++;
2250         is_add = 0;
2251         break;
2252       }
2253   }
2254
2255   if (is_add)
2256     {
2257       vec_add2 (sm->outside_fibs, outside_fib, 1);
2258       outside_fib->refcount = 1;
2259       outside_fib->fib_index = new_fib_index;
2260     }
2261 }
2262
2263 static void
2264 snat_ip4_table_bind (ip4_main_t * im,
2265                      uword opaque,
2266                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2267 {
2268   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2269 }
2270
2271 static void
2272 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2273                                        uword opaque,
2274                                        u32 sw_if_index,
2275                                        ip4_address_t * address,
2276                                        u32 address_length,
2277                                        u32 if_address_index, u32 is_delete);
2278
2279 static void
2280 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2281                                  uword opaque,
2282                                  u32 sw_if_index,
2283                                  ip4_address_t * address,
2284                                  u32 address_length,
2285                                  u32 if_address_index, u32 is_delete);
2286
2287 static int
2288 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2289                                  u32 fib_index,
2290                                  u32 thread_index,
2291                                  snat_session_key_t * k,
2292                                  u16 port_per_thread, u32 snat_thread_index);
2293
2294 static clib_error_t *
2295 snat_init (vlib_main_t * vm)
2296 {
2297   snat_main_t *sm = &snat_main;
2298   clib_error_t *error = 0;
2299   ip4_main_t *im = &ip4_main;
2300   ip_lookup_main_t *lm = &im->lookup_main;
2301   uword *p;
2302   vlib_thread_registration_t *tr;
2303   vlib_thread_main_t *tm = vlib_get_thread_main ();
2304   uword *bitmap = 0;
2305   u32 i;
2306   ip4_add_del_interface_address_callback_t cb4;
2307   vlib_node_t *node;
2308
2309   sm->vlib_main = vm;
2310   sm->vnet_main = vnet_get_main ();
2311   sm->ip4_main = im;
2312   sm->ip4_lookup_main = lm;
2313   sm->api_main = vlibapi_get_main ();
2314   sm->first_worker_index = 0;
2315   sm->num_workers = 0;
2316   sm->num_snat_thread = 1;
2317   sm->workers = 0;
2318   sm->port_per_thread = 0xffff - 1024;
2319   sm->fq_in2out_index = ~0;
2320   sm->fq_in2out_output_index = ~0;
2321   sm->fq_out2in_index = ~0;
2322
2323
2324   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2325   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2326   sm->forwarding_enabled = 0;
2327   sm->log_class = vlib_log_register_class ("nat", 0);
2328   sm->log_level = SNAT_LOG_ERROR;
2329   sm->mss_clamping = 0;
2330
2331   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2332   sm->error_node_index = node->index;
2333
2334   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2335   sm->pre_in2out_node_index = node->index;
2336   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2337   sm->pre_out2in_node_index = node->index;
2338
2339   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2340   sm->pre_in2out_node_index = node->index;
2341
2342   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2343   sm->pre_out2in_node_index = node->index;
2344
2345   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2346   sm->in2out_node_index = node->index;
2347   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2348   sm->in2out_output_node_index = node->index;
2349   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2350   sm->in2out_fast_node_index = node->index;
2351   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2352   sm->in2out_slowpath_node_index = node->index;
2353   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2354   sm->in2out_slowpath_output_node_index = node->index;
2355   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-reass");
2356   sm->in2out_reass_node_index = node->index;
2357
2358   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2359   sm->ed_in2out_node_index = node->index;
2360   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2361   sm->ed_in2out_slowpath_node_index = node->index;
2362   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-reass");
2363   sm->ed_in2out_reass_node_index = node->index;
2364
2365   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2366   sm->out2in_node_index = node->index;
2367   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2368   sm->out2in_fast_node_index = node->index;
2369   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-reass");
2370   sm->out2in_reass_node_index = node->index;
2371
2372   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2373   sm->ed_out2in_node_index = node->index;
2374   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2375   sm->ed_out2in_slowpath_node_index = node->index;
2376   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-reass");
2377   sm->ed_out2in_reass_node_index = node->index;
2378
2379   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2380   sm->det_in2out_node_index = node->index;
2381   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2382   sm->det_out2in_node_index = node->index;
2383
2384   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2385   sm->hairpinning_node_index = node->index;
2386   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2387   sm->hairpin_dst_node_index = node->index;
2388   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2389   sm->hairpin_src_node_index = node->index;
2390   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2391   sm->ed_hairpinning_node_index = node->index;
2392   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2393   sm->ed_hairpin_dst_node_index = node->index;
2394   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2395   sm->ed_hairpin_src_node_index = node->index;
2396
2397   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2398   if (p)
2399     {
2400       tr = (vlib_thread_registration_t *) p[0];
2401       if (tr)
2402         {
2403           sm->num_workers = tr->count;
2404           sm->first_worker_index = tr->first_index;
2405         }
2406     }
2407
2408   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2409
2410   /* Use all available workers by default */
2411   if (sm->num_workers > 1)
2412     {
2413       for (i = 0; i < sm->num_workers; i++)
2414         bitmap = clib_bitmap_set (bitmap, i, 1);
2415       snat_set_workers (bitmap);
2416       clib_bitmap_free (bitmap);
2417     }
2418   else
2419     {
2420       sm->per_thread_data[0].snat_thread_index = 0;
2421     }
2422
2423   error = snat_api_init (vm, sm);
2424   if (error)
2425     return error;
2426
2427   /* Set up the interface address add/del callback */
2428   cb4.function = snat_ip4_add_del_interface_address_cb;
2429   cb4.function_opaque = 0;
2430
2431   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2432
2433   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2434   cb4.function_opaque = 0;
2435
2436   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2437
2438   nat_dpo_module_init ();
2439
2440   /* Init counters */
2441   sm->total_users.name = "total-users";
2442   sm->total_users.stat_segment_name = "/nat44/total-users";
2443   vlib_validate_simple_counter (&sm->total_users, 0);
2444   vlib_zero_simple_counter (&sm->total_users, 0);
2445   sm->total_sessions.name = "total-sessions";
2446   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2447   vlib_validate_simple_counter (&sm->total_sessions, 0);
2448   vlib_zero_simple_counter (&sm->total_sessions, 0);
2449
2450   /* Init IPFIX logging */
2451   snat_ipfix_logging_init (vm);
2452
2453   /* Init NAT64 */
2454   error = nat64_init (vm);
2455   if (error)
2456     return error;
2457
2458   dslite_init (vm);
2459
2460   nat66_init (vm);
2461
2462   ip4_table_bind_callback_t cbt4 = {
2463     .function = snat_ip4_table_bind,
2464   };
2465   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2466
2467   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2468                                         FIB_SOURCE_PRIORITY_HI,
2469                                         FIB_SOURCE_BH_SIMPLE);
2470   nat_fib_src_low = fib_source_allocate ("nat-low",
2471                                          FIB_SOURCE_PRIORITY_LOW,
2472                                          FIB_SOURCE_BH_SIMPLE);
2473
2474   /* Init virtual fragmenentation reassembly */
2475   return nat_reass_init (vm);
2476 }
2477
2478 VLIB_INIT_FUNCTION (snat_init);
2479
2480 void
2481 snat_free_outside_address_and_port (snat_address_t * addresses,
2482                                     u32 thread_index, snat_session_key_t * k)
2483 {
2484   snat_address_t *a;
2485   u32 address_index;
2486   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2487
2488   for (address_index = 0; address_index < vec_len (addresses);
2489        address_index++)
2490     {
2491       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2492         break;
2493     }
2494
2495   ASSERT (address_index < vec_len (addresses));
2496
2497   a = addresses + address_index;
2498
2499   switch (k->protocol)
2500     {
2501 #define _(N, i, n, s) \
2502     case SNAT_PROTOCOL_##N: \
2503       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2504         port_host_byte_order) == 1); \
2505       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2506         port_host_byte_order, 0); \
2507       a->busy_##n##_ports--; \
2508       a->busy_##n##_ports_per_thread[thread_index]--; \
2509       break;
2510       foreach_snat_protocol
2511 #undef _
2512     default:
2513       nat_elog_info ("unknown protocol");
2514       return;
2515     }
2516 }
2517
2518 static int
2519 nat_set_outside_address_and_port (snat_address_t * addresses,
2520                                   u32 thread_index, snat_session_key_t * k)
2521 {
2522   snat_address_t *a = 0;
2523   u32 address_index;
2524   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2525
2526   for (address_index = 0; address_index < vec_len (addresses);
2527        address_index++)
2528     {
2529       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2530         continue;
2531
2532       a = addresses + address_index;
2533       switch (k->protocol)
2534         {
2535 #define _(N, j, n, s) \
2536         case SNAT_PROTOCOL_##N: \
2537           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2538             return VNET_API_ERROR_INSTANCE_IN_USE; \
2539           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2540           a->busy_##n##_ports_per_thread[thread_index]++; \
2541           a->busy_##n##_ports++; \
2542           return 0;
2543           foreach_snat_protocol
2544 #undef _
2545         default:
2546           nat_elog_info ("unknown protocol");
2547           return 1;
2548         }
2549     }
2550
2551   return VNET_API_ERROR_NO_SUCH_ENTRY;
2552 }
2553
2554 int
2555 snat_static_mapping_match (snat_main_t * sm,
2556                            snat_session_key_t match,
2557                            snat_session_key_t * mapping,
2558                            u8 by_external,
2559                            u8 * is_addr_only,
2560                            twice_nat_type_t * twice_nat,
2561                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2562                            u8 * is_identity_nat)
2563 {
2564   clib_bihash_kv_8_8_t kv, value;
2565   snat_static_mapping_t *m;
2566   snat_session_key_t m_key;
2567   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2568   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2569   u8 backend_index;
2570   nat44_lb_addr_port_t *local;
2571
2572   m_key.fib_index = match.fib_index;
2573   if (by_external)
2574     {
2575       mapping_hash = &sm->static_mapping_by_external;
2576       m_key.fib_index = 0;
2577     }
2578
2579   m_key.addr = match.addr;
2580   m_key.port = clib_net_to_host_u16 (match.port);
2581   m_key.protocol = match.protocol;
2582
2583   kv.key = m_key.as_u64;
2584
2585   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2586     {
2587       /* Try address only mapping */
2588       m_key.port = 0;
2589       m_key.protocol = 0;
2590       kv.key = m_key.as_u64;
2591       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2592         return 1;
2593     }
2594
2595   m = pool_elt_at_index (sm->static_mappings, value.value);
2596
2597   if (by_external)
2598     {
2599       if (is_lb_static_mapping (m))
2600         {
2601           if (PREDICT_FALSE (lb != 0))
2602             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2603           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2604                                                           match.addr,
2605                                                           match.protocol,
2606                                                           match.port,
2607                                                           &backend_index))
2608             {
2609               local = pool_elt_at_index (m->locals, backend_index);
2610               mapping->addr = local->addr;
2611               mapping->port = clib_host_to_net_u16 (local->port);
2612               mapping->fib_index = local->fib_index;
2613               goto end;
2614             }
2615           // pick locals matching this worker
2616           if (PREDICT_FALSE (sm->num_workers > 1))
2617             {
2618               u32 thread_index = vlib_get_thread_index ();
2619               /* *INDENT-OFF* */
2620               pool_foreach_index (i, m->locals,
2621               ({
2622                 local = pool_elt_at_index (m->locals, i);
2623
2624                 ip4_header_t ip = {
2625                   .src_address = local->addr,
2626                 };
2627
2628                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2629                     thread_index)
2630                   {
2631                     vec_add1 (tmp, i);
2632                   }
2633               }));
2634               /* *INDENT-ON* */
2635               ASSERT (vec_len (tmp) != 0);
2636             }
2637           else
2638             {
2639               /* *INDENT-OFF* */
2640               pool_foreach_index (i, m->locals,
2641               ({
2642                 vec_add1 (tmp, i);
2643               }));
2644               /* *INDENT-ON* */
2645             }
2646           hi = vec_len (tmp) - 1;
2647           local = pool_elt_at_index (m->locals, tmp[hi]);
2648           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2649           while (lo < hi)
2650             {
2651               mid = ((hi - lo) >> 1) + lo;
2652               local = pool_elt_at_index (m->locals, tmp[mid]);
2653               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2654             }
2655           local = pool_elt_at_index (m->locals, tmp[lo]);
2656           if (!(local->prefix >= rand))
2657             return 1;
2658           mapping->addr = local->addr;
2659           mapping->port = clib_host_to_net_u16 (local->port);
2660           mapping->fib_index = local->fib_index;
2661           if (m->affinity)
2662             {
2663               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2664                                                 match.protocol, match.port,
2665                                                 tmp[lo], m->affinity,
2666                                                 m->affinity_per_service_list_head_index))
2667                 nat_elog_info ("create affinity record failed");
2668             }
2669           vec_free (tmp);
2670         }
2671       else
2672         {
2673           if (PREDICT_FALSE (lb != 0))
2674             *lb = NO_LB_NAT;
2675           mapping->fib_index = m->fib_index;
2676           mapping->addr = m->local_addr;
2677           /* Address only mapping doesn't change port */
2678           mapping->port = is_addr_only_static_mapping (m) ? match.port
2679             : clib_host_to_net_u16 (m->local_port);
2680         }
2681       mapping->protocol = m->proto;
2682     }
2683   else
2684     {
2685       mapping->addr = m->external_addr;
2686       /* Address only mapping doesn't change port */
2687       mapping->port = is_addr_only_static_mapping (m) ? match.port
2688         : clib_host_to_net_u16 (m->external_port);
2689       mapping->fib_index = sm->outside_fib_index;
2690     }
2691
2692 end:
2693   if (PREDICT_FALSE (is_addr_only != 0))
2694     *is_addr_only = is_addr_only_static_mapping (m);
2695
2696   if (PREDICT_FALSE (twice_nat != 0))
2697     *twice_nat = m->twice_nat;
2698
2699   if (PREDICT_FALSE (is_identity_nat != 0))
2700     *is_identity_nat = is_identity_static_mapping (m);
2701
2702   return 0;
2703 }
2704
2705 static_always_inline u16
2706 snat_random_port (u16 min, u16 max)
2707 {
2708   snat_main_t *sm = &snat_main;
2709   return min + random_u32 (&sm->random_seed) /
2710     (random_u32_max () / (max - min + 1) + 1);
2711 }
2712
2713 int
2714 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2715                                      u32 fib_index,
2716                                      u32 thread_index,
2717                                      snat_session_key_t * k,
2718                                      u16 port_per_thread,
2719                                      u32 snat_thread_index)
2720 {
2721   snat_main_t *sm = &snat_main;
2722
2723   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2724                                   port_per_thread, snat_thread_index);
2725 }
2726
2727 static int
2728 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2729                                  u32 fib_index,
2730                                  u32 thread_index,
2731                                  snat_session_key_t * k,
2732                                  u16 port_per_thread, u32 snat_thread_index)
2733 {
2734   int i;
2735   snat_address_t *a, *ga = 0;
2736   u32 portnum;
2737
2738   for (i = 0; i < vec_len (addresses); i++)
2739     {
2740       a = addresses + i;
2741       switch (k->protocol)
2742         {
2743 #define _(N, j, n, s) \
2744         case SNAT_PROTOCOL_##N: \
2745           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2746             { \
2747               if (a->fib_index == fib_index) \
2748                 { \
2749                   while (1) \
2750                     { \
2751                       portnum = (port_per_thread * \
2752                         snat_thread_index) + \
2753                         snat_random_port(1, port_per_thread) + 1024; \
2754                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2755                         continue; \
2756                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2757                       a->busy_##n##_ports_per_thread[thread_index]++; \
2758                       a->busy_##n##_ports++; \
2759                       k->addr = a->addr; \
2760                       k->port = clib_host_to_net_u16(portnum); \
2761                       return 0; \
2762                     } \
2763                 } \
2764               else if (a->fib_index == ~0) \
2765                 { \
2766                   ga = a; \
2767                 } \
2768             } \
2769           break;
2770           foreach_snat_protocol
2771 #undef _
2772         default:
2773           nat_elog_info ("unknown protocol");
2774           return 1;
2775         }
2776
2777     }
2778
2779   if (ga)
2780     {
2781       a = ga;
2782       switch (k->protocol)
2783         {
2784 #define _(N, j, n, s) \
2785         case SNAT_PROTOCOL_##N: \
2786           while (1) \
2787             { \
2788               portnum = (port_per_thread * \
2789                 snat_thread_index) + \
2790                 snat_random_port(1, port_per_thread) + 1024; \
2791               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2792                 continue; \
2793               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2794               a->busy_##n##_ports_per_thread[thread_index]++; \
2795               a->busy_##n##_ports++; \
2796               k->addr = a->addr; \
2797               k->port = clib_host_to_net_u16(portnum); \
2798               return 0; \
2799             }
2800           break;
2801           foreach_snat_protocol
2802 #undef _
2803         default:
2804           nat_elog_info ("unknown protocol");
2805           return 1;
2806         }
2807     }
2808
2809   /* Totally out of translations to use... */
2810   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2811   return 1;
2812 }
2813
2814 static int
2815 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2816                               u32 fib_index,
2817                               u32 thread_index,
2818                               snat_session_key_t * k,
2819                               u16 port_per_thread, u32 snat_thread_index)
2820 {
2821   snat_main_t *sm = &snat_main;
2822   snat_address_t *a = addresses;
2823   u16 m, ports, portnum, A, j;
2824   m = 16 - (sm->psid_offset + sm->psid_length);
2825   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2826
2827   if (!vec_len (addresses))
2828     goto exhausted;
2829
2830   switch (k->protocol)
2831     {
2832 #define _(N, i, n, s) \
2833     case SNAT_PROTOCOL_##N: \
2834       if (a->busy_##n##_ports < ports) \
2835         { \
2836           while (1) \
2837             { \
2838               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2839               j = snat_random_port(0, pow2_mask(m)); \
2840               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2841               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2842                 continue; \
2843               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2844               a->busy_##n##_ports++; \
2845               k->addr = a->addr; \
2846               k->port = clib_host_to_net_u16 (portnum); \
2847               return 0; \
2848             } \
2849         } \
2850       break;
2851       foreach_snat_protocol
2852 #undef _
2853     default:
2854       nat_elog_info ("unknown protocol");
2855       return 1;
2856     }
2857
2858 exhausted:
2859   /* Totally out of translations to use... */
2860   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2861   return 1;
2862 }
2863
2864 static int
2865 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2866                                u32 fib_index,
2867                                u32 thread_index,
2868                                snat_session_key_t * k,
2869                                u16 port_per_thread, u32 snat_thread_index)
2870 {
2871   snat_main_t *sm = &snat_main;
2872   snat_address_t *a = addresses;
2873   u16 portnum, ports;
2874
2875   ports = sm->end_port - sm->start_port + 1;
2876
2877   if (!vec_len (addresses))
2878     goto exhausted;
2879
2880   switch (k->protocol)
2881     {
2882 #define _(N, i, n, s) \
2883     case SNAT_PROTOCOL_##N: \
2884       if (a->busy_##n##_ports < ports) \
2885         { \
2886           while (1) \
2887             { \
2888               portnum = snat_random_port(sm->start_port, sm->end_port); \
2889               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2890                 continue; \
2891               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2892               a->busy_##n##_ports++; \
2893               k->addr = a->addr; \
2894               k->port = clib_host_to_net_u16 (portnum); \
2895               return 0; \
2896             } \
2897         } \
2898       break;
2899       foreach_snat_protocol
2900 #undef _
2901     default:
2902       nat_elog_info ("unknown protocol");
2903       return 1;
2904     }
2905
2906 exhausted:
2907   /* Totally out of translations to use... */
2908   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2909   return 1;
2910 }
2911
2912 void
2913 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2914 {
2915   dpo_id_t dpo_v4 = DPO_INVALID;
2916   fib_prefix_t pfx = {
2917     .fp_proto = FIB_PROTOCOL_IP4,
2918     .fp_len = 32,
2919     .fp_addr.ip4.as_u32 = addr.as_u32,
2920   };
2921
2922   if (is_add)
2923     {
2924       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2925       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
2926                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2927       dpo_reset (&dpo_v4);
2928     }
2929   else
2930     {
2931       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
2932     }
2933 }
2934
2935 u8 *
2936 format_session_kvp (u8 * s, va_list * args)
2937 {
2938   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2939   snat_session_key_t k;
2940
2941   k.as_u64 = v->key;
2942
2943   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2944
2945   return s;
2946 }
2947
2948 u8 *
2949 format_static_mapping_kvp (u8 * s, va_list * args)
2950 {
2951   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2952   snat_session_key_t k;
2953
2954   k.as_u64 = v->key;
2955
2956   s = format (s, "%U static-mapping-index %llu",
2957               format_static_mapping_key, &k, v->value);
2958
2959   return s;
2960 }
2961
2962 u8 *
2963 format_user_kvp (u8 * s, va_list * args)
2964 {
2965   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2966   snat_user_key_t k;
2967
2968   k.as_u64 = v->key;
2969
2970   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2971               k.fib_index, v->value);
2972
2973   return s;
2974 }
2975
2976 u8 *
2977 format_ed_session_kvp (u8 * s, va_list * args)
2978 {
2979   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2980   nat_ed_ses_key_t k;
2981
2982   k.as_u64[0] = v->key[0];
2983   k.as_u64[1] = v->key[1];
2984
2985   s =
2986     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2987             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2988             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2989             format_ip_protocol, k.proto, k.fib_index, v->value);
2990
2991   return s;
2992 }
2993
2994 static u32
2995 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
2996                            u8 is_output)
2997 {
2998   snat_main_t *sm = &snat_main;
2999   u32 next_worker_index = 0;
3000   u32 hash;
3001
3002   next_worker_index = sm->first_worker_index;
3003   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3004     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3005
3006   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3007     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3008   else
3009     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3010
3011   return next_worker_index;
3012 }
3013
3014 static u32
3015 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3016                            u8 is_output)
3017 {
3018   snat_main_t *sm = &snat_main;
3019   udp_header_t *udp;
3020   u16 port;
3021   snat_session_key_t m_key;
3022   clib_bihash_kv_8_8_t kv, value;
3023   snat_static_mapping_t *m;
3024   u32 proto;
3025   u32 next_worker_index = 0;
3026
3027   /* first try static mappings without port */
3028   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3029     {
3030       m_key.addr = ip0->dst_address;
3031       m_key.port = 0;
3032       m_key.protocol = 0;
3033       m_key.fib_index = rx_fib_index0;
3034       kv.key = m_key.as_u64;
3035       if (!clib_bihash_search_8_8
3036           (&sm->static_mapping_by_external, &kv, &value))
3037         {
3038           m = pool_elt_at_index (sm->static_mappings, value.value);
3039           return m->workers[0];
3040         }
3041     }
3042
3043   proto = ip_proto_to_snat_proto (ip0->protocol);
3044   udp = ip4_next_header (ip0);
3045   port = udp->dst_port;
3046
3047   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
3048     {
3049       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
3050         return vlib_get_thread_index ();
3051
3052       nat_reass_ip4_t *reass;
3053       reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
3054                                   ip0->fragment_id, ip0->protocol);
3055
3056       if (reass && (reass->thread_index != (u32) ~ 0))
3057         return reass->thread_index;
3058
3059       if (ip4_is_first_fragment (ip0))
3060         {
3061           reass =
3062             nat_ip4_reass_create (ip0->src_address, ip0->dst_address,
3063                                   ip0->fragment_id, ip0->protocol);
3064           if (!reass)
3065             goto no_reass;
3066
3067           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3068             {
3069               m_key.addr = ip0->dst_address;
3070               m_key.port = clib_net_to_host_u16 (port);
3071               m_key.protocol = proto;
3072               m_key.fib_index = rx_fib_index0;
3073               kv.key = m_key.as_u64;
3074               if (!clib_bihash_search_8_8
3075                   (&sm->static_mapping_by_external, &kv, &value))
3076                 {
3077                   m = pool_elt_at_index (sm->static_mappings, value.value);
3078                   reass->thread_index = m->workers[0];
3079                   return reass->thread_index;
3080                 }
3081             }
3082           reass->thread_index = sm->first_worker_index;
3083           reass->thread_index +=
3084             sm->workers[(clib_net_to_host_u16 (port) - 1024) /
3085                         sm->port_per_thread];
3086           return reass->thread_index;
3087         }
3088       else
3089         return vlib_get_thread_index ();
3090     }
3091
3092 no_reass:
3093   /* unknown protocol */
3094   if (PREDICT_FALSE (proto == ~0))
3095     {
3096       /* use current thread */
3097       return vlib_get_thread_index ();
3098     }
3099
3100   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3101     {
3102       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3103       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3104       if (!icmp_is_error_message (icmp))
3105         port = echo->identifier;
3106       else
3107         {
3108           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3109           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3110           void *l4_header = ip4_next_header (inner_ip);
3111           switch (proto)
3112             {
3113             case SNAT_PROTOCOL_ICMP:
3114               icmp = (icmp46_header_t *) l4_header;
3115               echo = (icmp_echo_header_t *) (icmp + 1);
3116               port = echo->identifier;
3117               break;
3118             case SNAT_PROTOCOL_UDP:
3119             case SNAT_PROTOCOL_TCP:
3120               port = ((tcp_udp_header_t *) l4_header)->src_port;
3121               break;
3122             default:
3123               return vlib_get_thread_index ();
3124             }
3125         }
3126     }
3127
3128   /* try static mappings with port */
3129   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3130     {
3131       m_key.addr = ip0->dst_address;
3132       m_key.port = clib_net_to_host_u16 (port);
3133       m_key.protocol = proto;
3134       m_key.fib_index = rx_fib_index0;
3135       kv.key = m_key.as_u64;
3136       if (!clib_bihash_search_8_8
3137           (&sm->static_mapping_by_external, &kv, &value))
3138         {
3139           m = pool_elt_at_index (sm->static_mappings, value.value);
3140           return m->workers[0];
3141         }
3142     }
3143
3144   /* worker by outside port */
3145   next_worker_index = sm->first_worker_index;
3146   next_worker_index +=
3147     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3148   return next_worker_index;
3149 }
3150
3151 static u32
3152 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3153                                u8 is_output)
3154 {
3155   snat_main_t *sm = &snat_main;
3156   u32 next_worker_index = sm->first_worker_index;
3157   u32 hash;
3158
3159   clib_bihash_kv_16_8_t kv16, value16;
3160   snat_main_per_thread_data_t *tsm;
3161   udp_header_t *udp;
3162
3163   if (PREDICT_FALSE (is_output))
3164     {
3165       u32 fib_index = sm->outside_fib_index;
3166       nat_outside_fib_t *outside_fib;
3167       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3168       fib_prefix_t pfx = {
3169         .fp_proto = FIB_PROTOCOL_IP4,
3170         .fp_len = 32,
3171         .fp_addr = {
3172                     .ip4.as_u32 = ip->dst_address.as_u32,
3173                     }
3174         ,
3175       };
3176
3177       udp = ip4_next_header (ip);
3178
3179       switch (vec_len (sm->outside_fibs))
3180         {
3181         case 0:
3182           fib_index = sm->outside_fib_index;
3183           break;
3184         case 1:
3185           fib_index = sm->outside_fibs[0].fib_index;
3186           break;
3187         default:
3188             /* *INDENT-OFF* */
3189             vec_foreach (outside_fib, sm->outside_fibs)
3190               {
3191                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3192                 if (FIB_NODE_INDEX_INVALID != fei)
3193                   {
3194                     if (fib_entry_get_resolving_interface (fei) != ~0)
3195                       {
3196                         fib_index = outside_fib->fib_index;
3197                         break;
3198                       }
3199                   }
3200               }
3201             /* *INDENT-ON* */
3202           break;
3203         }
3204
3205       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3206                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3207
3208       /* *INDENT-OFF* */
3209       vec_foreach (tsm, sm->per_thread_data)
3210         {
3211           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3212                                                       &kv16, &value16)))
3213             {
3214               next_worker_index += tsm->thread_index;
3215
3216               nat_elog_debug_handoff (
3217                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3218                 next_worker_index, fib_index,
3219                 clib_net_to_host_u32 (ip->src_address.as_u32),
3220                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3221
3222               return next_worker_index;
3223             }
3224         }
3225       /* *INDENT-ON* */
3226     }
3227
3228   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3229     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3230
3231   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3232     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3233   else
3234     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3235
3236   if (PREDICT_TRUE (!is_output))
3237     {
3238       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3239                               next_worker_index, rx_fib_index,
3240                               clib_net_to_host_u32 (ip->src_address.as_u32),
3241                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3242     }
3243   else
3244     {
3245       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3246                               next_worker_index, rx_fib_index,
3247                               clib_net_to_host_u32 (ip->src_address.as_u32),
3248                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3249     }
3250
3251   return next_worker_index;
3252 }
3253
3254 static u32
3255 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index,
3256                                u8 is_output)
3257 {
3258   snat_main_t *sm = &snat_main;
3259   clib_bihash_kv_8_8_t kv, value;
3260   clib_bihash_kv_16_8_t kv16, value16;
3261   snat_main_per_thread_data_t *tsm;
3262
3263   u32 proto, next_worker_index = 0;
3264   udp_header_t *udp;
3265   u16 port;
3266   snat_static_mapping_t *m;
3267   u32 hash;
3268
3269   proto = ip_proto_to_snat_proto (ip->protocol);
3270
3271   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3272     {
3273       udp = ip4_next_header (ip);
3274
3275       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3276                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3277
3278       /* *INDENT-OFF* */
3279       vec_foreach (tsm, sm->per_thread_data)
3280         {
3281           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3282                                                       &kv16, &value16)))
3283             {
3284               next_worker_index = sm->first_worker_index + tsm->thread_index;
3285               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3286                           next_worker_index, rx_fib_index,
3287                           clib_net_to_host_u32 (ip->src_address.as_u32),
3288                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3289               return next_worker_index;
3290             }
3291           }
3292         /* *INDENT-ON* */
3293     }
3294   else if (proto == SNAT_PROTOCOL_ICMP)
3295     {
3296       nat_ed_ses_key_t key;
3297
3298       if (!get_icmp_o2i_ed_key (ip, &key))
3299         {
3300
3301           key.fib_index = rx_fib_index;
3302           kv16.key[0] = key.as_u64[0];
3303           kv16.key[1] = key.as_u64[1];
3304
3305           /* *INDENT-OFF* */
3306           vec_foreach (tsm, sm->per_thread_data)
3307             {
3308               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3309                                                           &kv16, &value16)))
3310                 {
3311                   next_worker_index = sm->first_worker_index +
3312                                       tsm->thread_index;
3313                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3314                               next_worker_index, rx_fib_index,
3315                               clib_net_to_host_u32 (ip->src_address.as_u32),
3316                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3317                   return next_worker_index;
3318                 }
3319             }
3320           /* *INDENT-ON* */
3321         }
3322     }
3323
3324   /* first try static mappings without port */
3325   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3326     {
3327       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3328       if (!clib_bihash_search_8_8
3329           (&sm->static_mapping_by_external, &kv, &value))
3330         {
3331           m = pool_elt_at_index (sm->static_mappings, value.value);
3332           next_worker_index = m->workers[0];
3333           goto done;
3334         }
3335     }
3336
3337   /* unknown protocol */
3338   if (PREDICT_FALSE (proto == ~0))
3339     {
3340       /* use current thread */
3341       next_worker_index = vlib_get_thread_index ();
3342       goto done;
3343     }
3344
3345   udp = ip4_next_header (ip);
3346   port = udp->dst_port;
3347
3348   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3349     {
3350       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3351       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3352       if (!icmp_is_error_message (icmp))
3353         port = echo->identifier;
3354       else
3355         {
3356           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3357           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3358           void *l4_header = ip4_next_header (inner_ip);
3359           switch (proto)
3360             {
3361             case SNAT_PROTOCOL_ICMP:
3362               icmp = (icmp46_header_t *) l4_header;
3363               echo = (icmp_echo_header_t *) (icmp + 1);
3364               port = echo->identifier;
3365               break;
3366             case SNAT_PROTOCOL_UDP:
3367             case SNAT_PROTOCOL_TCP:
3368               port = ((tcp_udp_header_t *) l4_header)->src_port;
3369               break;
3370             default:
3371               next_worker_index = vlib_get_thread_index ();
3372               goto done;
3373             }
3374         }
3375     }
3376
3377   /* try static mappings with port */
3378   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3379     {
3380       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3381                   clib_net_to_host_u16 (port));
3382       if (!clib_bihash_search_8_8
3383           (&sm->static_mapping_by_external, &kv, &value))
3384         {
3385           m = pool_elt_at_index (sm->static_mappings, value.value);
3386           if (!is_lb_static_mapping (m))
3387             {
3388               next_worker_index = m->workers[0];
3389               goto done;
3390             }
3391
3392           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3393             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3394
3395           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3396             next_worker_index =
3397               m->workers[hash & (_vec_len (m->workers) - 1)];
3398           else
3399             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3400           goto done;
3401         }
3402     }
3403
3404   /* worker by outside port */
3405   next_worker_index = sm->first_worker_index;
3406   next_worker_index +=
3407     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3408
3409 done:
3410   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3411                           clib_net_to_host_u32 (ip->src_address.as_u32),
3412                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3413   return next_worker_index;
3414 }
3415
3416 void
3417 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3418                 ip4_address_t * out_addr, u16 out_port,
3419                 ip4_address_t * eh_addr, u16 eh_port,
3420                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3421                 u32 fib_index, u16 flags, u32 thread_index)
3422 {
3423   snat_main_t *sm = &snat_main;
3424   snat_session_key_t key;
3425   snat_user_t *u;
3426   snat_session_t *s;
3427   clib_bihash_kv_8_8_t kv;
3428   f64 now = vlib_time_now (sm->vlib_main);
3429   nat_outside_fib_t *outside_fib;
3430   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3431   snat_main_per_thread_data_t *tsm;
3432   fib_prefix_t pfx = {
3433     .fp_proto = FIB_PROTOCOL_IP4,
3434     .fp_len = 32,
3435     .fp_addr = {
3436                 .ip4.as_u32 = eh_addr->as_u32,
3437                 },
3438   };
3439
3440   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3441
3442   key.addr.as_u32 = out_addr->as_u32;
3443   key.port = out_port;
3444   key.protocol = proto;
3445
3446   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3447     {
3448       if (nat_set_outside_address_and_port
3449           (sm->addresses, thread_index, &key))
3450         return;
3451     }
3452
3453   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3454   if (!u)
3455     return;
3456
3457   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3458   if (!s)
3459     return;
3460
3461   s->last_heard = now;
3462   s->flags = flags;
3463   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3464   s->ext_host_port = eh_port;
3465   user_session_increment (sm, u, snat_is_session_static (s));
3466   switch (vec_len (sm->outside_fibs))
3467     {
3468     case 0:
3469       key.fib_index = sm->outside_fib_index;
3470       break;
3471     case 1:
3472       key.fib_index = sm->outside_fibs[0].fib_index;
3473       break;
3474     default:
3475       /* *INDENT-OFF* */
3476       vec_foreach (outside_fib, sm->outside_fibs)
3477         {
3478           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3479           if (FIB_NODE_INDEX_INVALID != fei)
3480             {
3481               if (fib_entry_get_resolving_interface (fei) != ~0)
3482                 {
3483                   key.fib_index = outside_fib->fib_index;
3484                   break;
3485                 }
3486             }
3487         }
3488       /* *INDENT-ON* */
3489       break;
3490     }
3491   s->out2in = key;
3492   kv.key = key.as_u64;
3493   kv.value = s - tsm->sessions;
3494   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3495     nat_elog_warn ("out2in key add failed");
3496
3497   key.addr.as_u32 = in_addr->as_u32;
3498   key.port = in_port;
3499   key.fib_index = fib_index;
3500   s->in2out = key;
3501   kv.key = key.as_u64;
3502   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3503     nat_elog_warn ("in2out key add failed");
3504 }
3505
3506 void
3507 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3508                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3509                 u32 ti)
3510 {
3511   snat_main_t *sm = &snat_main;
3512   snat_session_key_t key;
3513   clib_bihash_kv_8_8_t kv, value;
3514   u32 thread_index;
3515   snat_session_t *s;
3516   snat_main_per_thread_data_t *tsm;
3517
3518   if (sm->num_workers > 1)
3519     thread_index =
3520       sm->first_worker_index +
3521       (sm->workers[(clib_net_to_host_u16 (out_port) -
3522                     1024) / sm->port_per_thread]);
3523   else
3524     thread_index = sm->num_workers;
3525   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3526
3527   key.addr.as_u32 = out_addr->as_u32;
3528   key.port = out_port;
3529   key.protocol = proto;
3530   key.fib_index = fib_index;
3531   kv.key = key.as_u64;
3532   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3533     return;
3534
3535   s = pool_elt_at_index (tsm->sessions, value.value);
3536   nat_free_session_data (sm, s, thread_index, 1);
3537   nat44_delete_session (sm, s, thread_index);
3538 }
3539
3540 void
3541 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3542                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3543                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3544 {
3545   snat_main_t *sm = &snat_main;
3546   snat_session_key_t key;
3547   clib_bihash_kv_8_8_t kv, value;
3548   snat_session_t *s;
3549   snat_main_per_thread_data_t *tsm;
3550
3551   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3552
3553   key.addr.as_u32 = out_addr->as_u32;
3554   key.port = out_port;
3555   key.protocol = proto;
3556   key.fib_index = fib_index;
3557   kv.key = key.as_u64;
3558   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3559     return;
3560
3561   s = pool_elt_at_index (tsm->sessions, value.value);
3562   s->total_pkts = total_pkts;
3563   s->total_bytes = total_bytes;
3564 }
3565
3566 void
3567 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3568                    ip4_address_t * out_addr, u16 out_port,
3569                    ip4_address_t * eh_addr, u16 eh_port,
3570                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3571                    u32 fib_index, u16 flags, u32 thread_index)
3572 {
3573   snat_main_t *sm = &snat_main;
3574   snat_session_key_t key;
3575   snat_user_t *u;
3576   snat_session_t *s;
3577   clib_bihash_kv_16_8_t kv;
3578   f64 now = vlib_time_now (sm->vlib_main);
3579   nat_outside_fib_t *outside_fib;
3580   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3581   snat_main_per_thread_data_t *tsm;
3582   fib_prefix_t pfx = {
3583     .fp_proto = FIB_PROTOCOL_IP4,
3584     .fp_len = 32,
3585     .fp_addr = {
3586                 .ip4.as_u32 = eh_addr->as_u32,
3587                 },
3588   };
3589
3590   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3591
3592   key.addr.as_u32 = out_addr->as_u32;
3593   key.port = out_port;
3594   key.protocol = proto;
3595
3596   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3597     {
3598       if (nat_set_outside_address_and_port
3599           (sm->addresses, thread_index, &key))
3600         return;
3601     }
3602
3603   key.addr.as_u32 = ehn_addr->as_u32;
3604   key.port = ehn_port;
3605   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3606     {
3607       if (nat_set_outside_address_and_port
3608           (sm->twice_nat_addresses, thread_index, &key))
3609         return;
3610     }
3611
3612   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3613   if (!u)
3614     return;
3615
3616   s = nat_ed_session_alloc (sm, u, thread_index, now);
3617   if (!s)
3618     return;
3619
3620   s->last_heard = now;
3621   s->flags = flags;
3622   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3623   s->ext_host_nat_port = s->ext_host_port = eh_port;
3624   if (is_twice_nat_session (s))
3625     {
3626       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3627       s->ext_host_nat_port = ehn_port;
3628     }
3629   user_session_increment (sm, u, snat_is_session_static (s));
3630   switch (vec_len (sm->outside_fibs))
3631     {
3632     case 0:
3633       key.fib_index = sm->outside_fib_index;
3634       break;
3635     case 1:
3636       key.fib_index = sm->outside_fibs[0].fib_index;
3637       break;
3638     default:
3639       /* *INDENT-OFF* */
3640       vec_foreach (outside_fib, sm->outside_fibs)
3641         {
3642           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3643           if (FIB_NODE_INDEX_INVALID != fei)
3644             {
3645               if (fib_entry_get_resolving_interface (fei) != ~0)
3646                 {
3647                   key.fib_index = outside_fib->fib_index;
3648                   break;
3649                 }
3650             }
3651         }
3652       /* *INDENT-ON* */
3653       break;
3654     }
3655   key.addr.as_u32 = out_addr->as_u32;
3656   key.port = out_port;
3657   s->out2in = key;
3658   kv.value = s - tsm->sessions;
3659
3660   key.addr.as_u32 = in_addr->as_u32;
3661   key.port = in_port;
3662   key.fib_index = fib_index;
3663   s->in2out = key;
3664
3665   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3666               snat_proto_to_ip_proto (proto), fib_index, in_port,
3667               s->ext_host_nat_port);
3668   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3669     nat_elog_warn ("in2out key add failed");
3670
3671   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3672               s->out2in.fib_index, out_port, eh_port);
3673   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3674     nat_elog_warn ("out2in key add failed");
3675 }
3676
3677 void
3678 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3679                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3680                    u32 fib_index, u32 ti)
3681 {
3682   snat_main_t *sm = &snat_main;
3683   nat_ed_ses_key_t key;
3684   clib_bihash_kv_16_8_t kv, value;
3685   u32 thread_index;
3686   snat_session_t *s;
3687   snat_main_per_thread_data_t *tsm;
3688
3689   if (sm->num_workers > 1)
3690     thread_index =
3691       sm->first_worker_index +
3692       (sm->workers[(clib_net_to_host_u16 (out_port) -
3693                     1024) / sm->port_per_thread]);
3694   else
3695     thread_index = sm->num_workers;
3696   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3697
3698   key.l_addr.as_u32 = out_addr->as_u32;
3699   key.l_port = out_port;
3700   key.r_addr.as_u32 = eh_addr->as_u32;
3701   key.r_port = eh_port;
3702   key.proto = proto;
3703   key.fib_index = fib_index;
3704   kv.key[0] = key.as_u64[0];
3705   kv.key[1] = key.as_u64[1];
3706   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3707     return;
3708
3709   s = pool_elt_at_index (tsm->sessions, value.value);
3710   nat_free_session_data (sm, s, thread_index, 1);
3711   nat44_delete_session (sm, s, thread_index);
3712 }
3713
3714 void
3715 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3716                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3717                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3718                    u32 thread_index)
3719 {
3720   snat_main_t *sm = &snat_main;
3721   nat_ed_ses_key_t key;
3722   clib_bihash_kv_16_8_t kv, value;
3723   snat_session_t *s;
3724   snat_main_per_thread_data_t *tsm;
3725
3726   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3727
3728   key.l_addr.as_u32 = out_addr->as_u32;
3729   key.l_port = out_port;
3730   key.r_addr.as_u32 = eh_addr->as_u32;
3731   key.r_port = eh_port;
3732   key.proto = proto;
3733   key.fib_index = fib_index;
3734   kv.key[0] = key.as_u64[0];
3735   kv.key[1] = key.as_u64[1];
3736   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3737     return;
3738
3739   s = pool_elt_at_index (tsm->sessions, value.value);
3740   s->total_pkts = total_pkts;
3741   s->total_bytes = total_bytes;
3742 }
3743
3744 static clib_error_t *
3745 snat_config (vlib_main_t * vm, unformat_input_t * input)
3746 {
3747   snat_main_t *sm = &snat_main;
3748   nat66_main_t *nm = &nat66_main;
3749   dslite_main_t *dm = &dslite_main;
3750   snat_main_per_thread_data_t *tsm;
3751
3752   u32 static_mapping_buckets = 1024;
3753   u32 static_mapping_memory_size = 64 << 20;
3754
3755   u32 nat64_bib_buckets = 1024;
3756   u32 nat64_bib_memory_size = 128 << 20;
3757
3758   u32 nat64_st_buckets = 2048;
3759   u32 nat64_st_memory_size = 256 << 20;
3760
3761   u32 user_buckets = 128;
3762   u32 user_memory_size = 64 << 20;
3763   u32 translation_buckets = 1024;
3764   u32 translation_memory_size = 128 << 20;
3765
3766   u32 max_translations_per_user = ~0;
3767
3768   u32 outside_vrf_id = 0;
3769   u32 outside_ip6_vrf_id = 0;
3770   u32 inside_vrf_id = 0;
3771   u8 static_mapping_only = 0;
3772   u8 static_mapping_connection_tracking = 0;
3773
3774   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3775   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3776
3777   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3778   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3779
3780   sm->deterministic = 0;
3781   sm->out2in_dpo = 0;
3782   sm->endpoint_dependent = 0;
3783
3784   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3785     {
3786       if (unformat
3787           (input, "translation hash buckets %d", &translation_buckets))
3788         ;
3789       else if (unformat (input, "udp timeout %d", &udp_timeout))
3790         ;
3791       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3792         ;
3793       else if (unformat (input, "tcp transitory timeout %d",
3794                          &tcp_transitory_timeout));
3795       else if (unformat (input, "tcp established timeout %d",
3796                          &tcp_established_timeout));
3797       else if (unformat (input, "translation hash memory %d",
3798                          &translation_memory_size));
3799       else if (unformat (input, "user hash buckets %d", &user_buckets))
3800         ;
3801       else if (unformat (input, "user hash memory %d", &user_memory_size))
3802         ;
3803       else if (unformat (input, "max translations per user %d",
3804                          &max_translations_per_user))
3805         ;
3806       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3807         ;
3808       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3809         ;
3810       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3811         ;
3812       else if (unformat (input, "static mapping only"))
3813         {
3814           static_mapping_only = 1;
3815           if (unformat (input, "connection tracking"))
3816             static_mapping_connection_tracking = 1;
3817         }
3818       else if (unformat (input, "deterministic"))
3819         sm->deterministic = 1;
3820       else if (unformat (input, "nat64 bib hash buckets %d",
3821                          &nat64_bib_buckets))
3822         ;
3823       else if (unformat (input, "nat64 bib hash memory %d",
3824                          &nat64_bib_memory_size))
3825         ;
3826       else
3827         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3828         ;
3829       else if (unformat (input, "nat64 st hash memory %d",
3830                          &nat64_st_memory_size))
3831         ;
3832       else if (unformat (input, "out2in dpo"))
3833         sm->out2in_dpo = 1;
3834       else if (unformat (input, "dslite ce"))
3835         dslite_set_ce (dm, 1);
3836       else if (unformat (input, "endpoint-dependent"))
3837         sm->endpoint_dependent = 1;
3838       else
3839         return clib_error_return (0, "unknown input '%U'",
3840                                   format_unformat_error, input);
3841     }
3842
3843   if (sm->deterministic && sm->endpoint_dependent)
3844     return clib_error_return (0,
3845                               "deterministic and endpoint-dependent modes are mutually exclusive");
3846
3847   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3848     return clib_error_return (0,
3849                               "static mapping only mode available only for simple nat");
3850
3851   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3852     return clib_error_return (0,
3853                               "out2in dpo mode available only for simple nat");
3854
3855   /* optionally configurable timeouts for testing purposes */
3856   sm->udp_timeout = udp_timeout;
3857   sm->icmp_timeout = icmp_timeout;
3858   sm->tcp_transitory_timeout = tcp_transitory_timeout;
3859   sm->tcp_established_timeout = tcp_established_timeout;
3860
3861   sm->user_buckets = user_buckets;
3862   sm->user_memory_size = user_memory_size;
3863
3864   sm->translation_buckets = translation_buckets;
3865   sm->translation_memory_size = translation_memory_size;
3866
3867   /* do not exceed load factor 10 */
3868   sm->max_translations = 10 * translation_buckets;
3869   sm->max_translations_per_user = max_translations_per_user == ~0 ?
3870     sm->max_translations : max_translations_per_user;
3871
3872   sm->outside_vrf_id = outside_vrf_id;
3873   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3874                                                              outside_vrf_id,
3875                                                              nat_fib_src_hi);
3876   nm->outside_vrf_id = outside_ip6_vrf_id;
3877   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3878                                                              outside_ip6_vrf_id,
3879                                                              nat_fib_src_hi);
3880   sm->inside_vrf_id = inside_vrf_id;
3881   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3882                                                             inside_vrf_id,
3883                                                             nat_fib_src_hi);
3884   sm->static_mapping_only = static_mapping_only;
3885   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3886
3887   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3888                   nat64_st_memory_size);
3889
3890   if (sm->deterministic)
3891     {
3892       sm->in2out_node_index = snat_det_in2out_node.index;
3893       sm->in2out_output_node_index = ~0;
3894       sm->out2in_node_index = snat_det_out2in_node.index;
3895       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3896       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3897     }
3898   else
3899     {
3900       if (sm->endpoint_dependent)
3901         {
3902           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
3903           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3904
3905           sm->handoff_out2in_index = nat_pre_out2in_node.index;
3906           sm->handoff_in2out_index = nat_pre_in2out_node.index;
3907           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
3908
3909           sm->in2out_node_index = nat44_ed_in2out_node.index;
3910           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3911           sm->out2in_node_index = nat44_ed_out2in_node.index;
3912
3913           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3914           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3915           nat_affinity_init (vm);
3916           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
3917                        nat_ha_sref_ed_cb);
3918         }
3919       else
3920         {
3921           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3922           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3923
3924           sm->handoff_out2in_index = snat_in2out_node.index;
3925           sm->handoff_in2out_index = snat_out2in_node.index;
3926           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
3927
3928           sm->in2out_node_index = snat_in2out_node.index;
3929           sm->in2out_output_node_index = snat_in2out_output_node.index;
3930           sm->out2in_node_index = snat_out2in_node.index;
3931           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3932           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3933           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
3934         }
3935       if (!static_mapping_only ||
3936           (static_mapping_only && static_mapping_connection_tracking))
3937         {
3938           /* *INDENT-OFF* */
3939           vec_foreach (tsm, sm->per_thread_data)
3940             {
3941               if (sm->endpoint_dependent)
3942                 {
3943                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3944                                          translation_buckets,
3945                                          translation_memory_size);
3946                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3947                                                       format_ed_session_kvp);
3948
3949                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3950                                          translation_buckets,
3951                                          translation_memory_size);
3952                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3953                                                       format_ed_session_kvp);
3954                 }
3955               else
3956                 {
3957                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3958                                         translation_buckets,
3959                                         translation_memory_size);
3960                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3961                                                      format_session_kvp);
3962
3963                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3964                                         translation_buckets,
3965                                         translation_memory_size);
3966                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3967                                                      format_session_kvp);
3968                 }
3969
3970               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3971                                     user_memory_size);
3972               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3973                                                  format_user_kvp);
3974             }
3975           /* *INDENT-ON* */
3976
3977         }
3978       else
3979         {
3980           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3981           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3982         }
3983       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3984                             "static_mapping_by_local", static_mapping_buckets,
3985                             static_mapping_memory_size);
3986       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3987                                          format_static_mapping_kvp);
3988
3989       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3990                             "static_mapping_by_external",
3991                             static_mapping_buckets,
3992                             static_mapping_memory_size);
3993       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3994                                          format_static_mapping_kvp);
3995     }
3996
3997   return 0;
3998 }
3999
4000 VLIB_CONFIG_FUNCTION (snat_config, "nat");
4001
4002 static void
4003 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4004                                  uword opaque,
4005                                  u32 sw_if_index,
4006                                  ip4_address_t * address,
4007                                  u32 address_length,
4008                                  u32 if_address_index, u32 is_delete)
4009 {
4010   snat_main_t *sm = &snat_main;
4011   snat_static_map_resolve_t *rp;
4012   snat_static_mapping_t *m;
4013   snat_session_key_t m_key;
4014   clib_bihash_kv_8_8_t kv, value;
4015   int i, rv;
4016   ip4_address_t l_addr;
4017
4018   for (i = 0; i < vec_len (sm->to_resolve); i++)
4019     {
4020       rp = sm->to_resolve + i;
4021       if (rp->addr_only == 0)
4022         continue;
4023       if (rp->sw_if_index == sw_if_index)
4024         goto match;
4025     }
4026
4027   return;
4028
4029 match:
4030   m_key.addr.as_u32 = address->as_u32;
4031   m_key.port = rp->addr_only ? 0 : rp->e_port;
4032   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4033   m_key.fib_index = sm->outside_fib_index;
4034   kv.key = m_key.as_u64;
4035   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4036     m = 0;
4037   else
4038     m = pool_elt_at_index (sm->static_mappings, value.value);
4039
4040   if (!is_delete)
4041     {
4042       /* Don't trip over lease renewal, static config */
4043       if (m)
4044         return;
4045     }
4046   else
4047     {
4048       if (!m)
4049         return;
4050     }
4051
4052   /* Indetity mapping? */
4053   if (rp->l_addr.as_u32 == 0)
4054     l_addr.as_u32 = address[0].as_u32;
4055   else
4056     l_addr.as_u32 = rp->l_addr.as_u32;
4057   /* Add the static mapping */
4058   rv = snat_add_static_mapping (l_addr,
4059                                 address[0],
4060                                 rp->l_port,
4061                                 rp->e_port,
4062                                 rp->vrf_id,
4063                                 rp->addr_only, ~0 /* sw_if_index */ ,
4064                                 rp->proto, !is_delete, rp->twice_nat,
4065                                 rp->out2in_only, rp->tag, rp->identity_nat);
4066   if (rv)
4067     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4068 }
4069
4070 static void
4071 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4072                                        uword opaque,
4073                                        u32 sw_if_index,
4074                                        ip4_address_t * address,
4075                                        u32 address_length,
4076                                        u32 if_address_index, u32 is_delete)
4077 {
4078   snat_main_t *sm = &snat_main;
4079   snat_static_map_resolve_t *rp;
4080   ip4_address_t l_addr;
4081   int i, j;
4082   int rv;
4083   u8 twice_nat = 0;
4084   snat_address_t *addresses = sm->addresses;
4085
4086   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4087     {
4088       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4089         goto match;
4090     }
4091
4092   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4093     {
4094       twice_nat = 1;
4095       addresses = sm->twice_nat_addresses;
4096       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4097         goto match;
4098     }
4099
4100   return;
4101
4102 match:
4103   if (!is_delete)
4104     {
4105       /* Don't trip over lease renewal, static config */
4106       for (j = 0; j < vec_len (addresses); j++)
4107         if (addresses[j].addr.as_u32 == address->as_u32)
4108           return;
4109
4110       (void) snat_add_address (sm, address, ~0, twice_nat);
4111       /* Scan static map resolution vector */
4112       for (j = 0; j < vec_len (sm->to_resolve); j++)
4113         {
4114           rp = sm->to_resolve + j;
4115           if (rp->addr_only)
4116             continue;
4117           /* On this interface? */
4118           if (rp->sw_if_index == sw_if_index)
4119             {
4120               /* Indetity mapping? */
4121               if (rp->l_addr.as_u32 == 0)
4122                 l_addr.as_u32 = address[0].as_u32;
4123               else
4124                 l_addr.as_u32 = rp->l_addr.as_u32;
4125               /* Add the static mapping */
4126               rv = snat_add_static_mapping (l_addr,
4127                                             address[0],
4128                                             rp->l_port,
4129                                             rp->e_port,
4130                                             rp->vrf_id,
4131                                             rp->addr_only,
4132                                             ~0 /* sw_if_index */ ,
4133                                             rp->proto,
4134                                             rp->is_add, rp->twice_nat,
4135                                             rp->out2in_only, rp->tag,
4136                                             rp->identity_nat);
4137               if (rv)
4138                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4139                                     "i4", rv);
4140             }
4141         }
4142       return;
4143     }
4144   else
4145     {
4146       (void) snat_del_address (sm, address[0], 1, twice_nat);
4147       return;
4148     }
4149 }
4150
4151
4152 int
4153 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4154                             u8 twice_nat)
4155 {
4156   ip4_main_t *ip4_main = sm->ip4_main;
4157   ip4_address_t *first_int_addr;
4158   snat_static_map_resolve_t *rp;
4159   u32 *indices_to_delete = 0;
4160   int i, j;
4161   u32 *auto_add_sw_if_indices =
4162     twice_nat ? sm->
4163     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4164
4165   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4166     );
4167
4168   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4169     {
4170       if (auto_add_sw_if_indices[i] == sw_if_index)
4171         {
4172           if (is_del)
4173             {
4174               /* if have address remove it */
4175               if (first_int_addr)
4176                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4177               else
4178                 {
4179                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4180                     {
4181                       rp = sm->to_resolve + j;
4182                       if (rp->sw_if_index == sw_if_index)
4183                         vec_add1 (indices_to_delete, j);
4184                     }
4185                   if (vec_len (indices_to_delete))
4186                     {
4187                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4188                         vec_del1 (sm->to_resolve, j);
4189                       vec_free (indices_to_delete);
4190                     }
4191                 }
4192               if (twice_nat)
4193                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4194               else
4195                 vec_del1 (sm->auto_add_sw_if_indices, i);
4196             }
4197           else
4198             return VNET_API_ERROR_VALUE_EXIST;
4199
4200           return 0;
4201         }
4202     }
4203
4204   if (is_del)
4205     return VNET_API_ERROR_NO_SUCH_ENTRY;
4206
4207   /* add to the auto-address list */
4208   if (twice_nat)
4209     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4210   else
4211     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4212
4213   /* If the address is already bound - or static - add it now */
4214   if (first_int_addr)
4215     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4216
4217   return 0;
4218 }
4219
4220 int
4221 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4222                    snat_protocol_t proto, u32 vrf_id, int is_in)
4223 {
4224   snat_main_per_thread_data_t *tsm;
4225   clib_bihash_kv_8_8_t kv, value;
4226   ip4_header_t ip;
4227   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4228   snat_session_key_t key;
4229   snat_session_t *s;
4230   clib_bihash_8_8_t *t;
4231
4232   if (sm->endpoint_dependent)
4233     return VNET_API_ERROR_UNSUPPORTED;
4234
4235   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4236   if (sm->num_workers > 1)
4237     tsm =
4238       vec_elt_at_index (sm->per_thread_data,
4239                         sm->worker_in2out_cb (&ip, fib_index, 0));
4240   else
4241     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4242
4243   key.addr.as_u32 = addr->as_u32;
4244   key.port = clib_host_to_net_u16 (port);
4245   key.protocol = proto;
4246   key.fib_index = fib_index;
4247   kv.key = key.as_u64;
4248   t = is_in ? &tsm->in2out : &tsm->out2in;
4249   if (!clib_bihash_search_8_8 (t, &kv, &value))
4250     {
4251       if (pool_is_free_index (tsm->sessions, value.value))
4252         return VNET_API_ERROR_UNSPECIFIED;
4253
4254       s = pool_elt_at_index (tsm->sessions, value.value);
4255       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4256       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4257       return 0;
4258     }
4259
4260   return VNET_API_ERROR_NO_SUCH_ENTRY;
4261 }
4262
4263 int
4264 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4265                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4266                       u32 vrf_id, int is_in)
4267 {
4268   ip4_header_t ip;
4269   clib_bihash_16_8_t *t;
4270   nat_ed_ses_key_t key;
4271   clib_bihash_kv_16_8_t kv, value;
4272   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4273   snat_session_t *s;
4274   snat_main_per_thread_data_t *tsm;
4275
4276   if (!sm->endpoint_dependent)
4277     return VNET_API_ERROR_FEATURE_DISABLED;
4278
4279   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4280   if (sm->num_workers > 1)
4281     tsm =
4282       vec_elt_at_index (sm->per_thread_data,
4283                         sm->worker_in2out_cb (&ip, fib_index, 0));
4284   else
4285     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4286
4287   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4288   key.l_addr.as_u32 = addr->as_u32;
4289   key.r_addr.as_u32 = eh_addr->as_u32;
4290   key.l_port = clib_host_to_net_u16 (port);
4291   key.r_port = clib_host_to_net_u16 (eh_port);
4292   key.proto = proto;
4293   key.fib_index = fib_index;
4294   kv.key[0] = key.as_u64[0];
4295   kv.key[1] = key.as_u64[1];
4296   if (clib_bihash_search_16_8 (t, &kv, &value))
4297     return VNET_API_ERROR_NO_SUCH_ENTRY;
4298
4299   if (pool_is_free_index (tsm->sessions, value.value))
4300     return VNET_API_ERROR_UNSPECIFIED;
4301   s = pool_elt_at_index (tsm->sessions, value.value);
4302   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4303   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4304   return 0;
4305 }
4306
4307 void
4308 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4309 {
4310   snat_main_t *sm = &snat_main;
4311
4312   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4313   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4314   sm->psid = psid;
4315   sm->psid_offset = psid_offset;
4316   sm->psid_length = psid_length;
4317 }
4318
4319 void
4320 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4321 {
4322   snat_main_t *sm = &snat_main;
4323
4324   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4325   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4326   sm->start_port = start_port;
4327   sm->end_port = end_port;
4328 }
4329
4330 void
4331 nat_set_alloc_addr_and_port_default (void)
4332 {
4333   snat_main_t *sm = &snat_main;
4334
4335   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4336   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4337 }
4338
4339 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4340                                  vlib_node_runtime_t * node,
4341                                  vlib_frame_t * frame)
4342 {
4343   return 0;
4344 }
4345
4346 /* *INDENT-OFF* */
4347 VLIB_REGISTER_NODE (nat_default_node) = {
4348   .name = "nat-default",
4349   .vector_size = sizeof (u32),
4350   .format_trace = 0,
4351   .type = VLIB_NODE_TYPE_INTERNAL,
4352   .n_errors = 0,
4353   .n_next_nodes = NAT_N_NEXT,
4354   .next_nodes = {
4355     [NAT_NEXT_DROP] = "error-drop",
4356     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4357     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4358     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4359     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4360     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4361     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4362     [NAT_NEXT_IN2OUT_ED_REASS] = "nat44-ed-in2out-reass",
4363     [NAT_NEXT_IN2OUT_ED_OUTPUT_REASS] = "nat44-ed-in2out-reass-output",
4364     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4365     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4366     [NAT_NEXT_OUT2IN_ED_REASS] = "nat44-ed-out2in-reass",
4367   },
4368 };
4369 /* *INDENT-ON* */
4370
4371 /*
4372  * fd.io coding-style-patch-verification: ON
4373  *
4374  * Local Variables:
4375  * eval: (c-set-style "gnu")
4376  * End:
4377  */